Skip to content

Commit 29d96d3

Browse files
keshav-spacekeshav-space
committed
Add step to import newly collected advisory
Signed-off-by: Keshav Priyadarshi <git@keshav.space>
1 parent 48e8527 commit 29d96d3

2 files changed

Lines changed: 177 additions & 34 deletions

File tree

‎vulnerabilities/pipelines/__init__.py‎

Lines changed: 18 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,10 @@
1515
from aboutcode.pipeline import BasePipeline
1616
from aboutcode.pipeline import LoopProgress
1717

18-
from vulnerabilities import import_runner
1918
from vulnerabilities.importer import AdvisoryData
20-
from vulnerabilities.improvers.default import DefaultImporter
21-
from vulnerabilities.models import Advisory
19+
from vulnerabilities.improver import MAX_CONFIDENCE
20+
from vulnerabilities.pipelines.pipes.importer import import_advisory
21+
from vulnerabilities.pipelines.pipes.importer import insert_advisory
2222
from vulnerabilities.utils import classproperty
2323

2424
module_logger = logging.getLogger(__name__)
@@ -47,13 +47,14 @@ class VulnerableCodeBaseImporterPipeline(VulnerableCodePipeline):
4747
4848
Uses:
4949
Subclass this Pipeline and implement ``advisories_count`` and ``collect_advisories`` method.
50-
Also override the ``steps`` if needed.
50+
Also override the ``steps`` and ``advisory_confidence`` as needed.
5151
"""
5252

5353
license_url = None
5454
spdx_license_expression = None
5555
repo_url = None
5656
importer_name = None
57+
advisory_confidence = MAX_CONFIDENCE
5758

5859
@classmethod
5960
def steps(cls):
@@ -86,34 +87,17 @@ def collect_and_store_advisories(self):
8687
collected_advisory_count = 0
8788
progress = LoopProgress(total_iterations=self.advisories_count(), logger=self.log)
8889
for advisory in progress.iter(self.collect_advisories()):
89-
self.insert_advisory(advisory=advisory)
90+
new_advisory = insert_advisory(
91+
advisory=advisory,
92+
pipeline_name=self.qualified_name,
93+
logger=self.log,
94+
)
95+
if new_advisory:
96+
self.new_advisories.append(new_advisory)
9097
collected_advisory_count += 1
9198

9299
self.log(f"Successfully collected {collected_advisory_count:,d} advisories")
93100

94-
def insert_advisory(self, advisory: AdvisoryData):
95-
try:
96-
obj, created = Advisory.objects.get_or_create(
97-
aliases=advisory.aliases,
98-
summary=advisory.summary,
99-
affected_packages=[pkg.to_dict() for pkg in advisory.affected_packages],
100-
references=[ref.to_dict() for ref in advisory.references],
101-
date_published=advisory.date_published,
102-
weaknesses=advisory.weaknesses,
103-
defaults={
104-
"created_by": self.qualified_name,
105-
"date_collected": datetime.now(timezone.utc),
106-
},
107-
url=advisory.url,
108-
)
109-
if created:
110-
self.new_advisories.append(obj)
111-
except Exception as e:
112-
self.log(
113-
f"Error while processing {advisory!r} with aliases {advisory.aliases!r}: {e!r} \n {traceback_format_exc()}",
114-
level=logging.ERROR,
115-
)
116-
117101
def import_new_advisories(self):
118102
new_advisories_count = len(self.new_advisories)
119103

@@ -129,14 +113,14 @@ def import_advisory(self, advisory) -> None:
129113
if advisory.date_imported:
130114
return
131115
try:
132-
advisory_importer = DefaultImporter(advisories=[advisory])
133-
inferences = advisory_importer.get_inferences(advisory_data=advisory.to_advisory_data())
134-
import_runner.process_inferences(
135-
inferences=inferences,
116+
import_advisory(
136117
advisory=advisory,
137-
improver_name=self.qualified_name,
118+
pipeline_name=self.qualified_name,
119+
confidence=self.advisory_confidence,
120+
logger=self.log,
138121
)
139122
except Exception as e:
140123
self.log(
141-
f"Failed to process advisory: {advisory!r} with error {e!r}", level=logging.ERROR
124+
f"Failed to process advisory: {advisory!r} with error {e!r}",
125+
level=logging.ERROR,
142126
)

‎vulnerabilities/pipelines/pipes/importer.py‎

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
#
2+
# Copyright (c) nexB Inc. and others. All rights reserved.
3+
# VulnerableCode is a trademark of nexB Inc.
4+
# SPDX-License-Identifier: Apache-2.0
5+
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
6+
# See https://github.com/nexB/vulnerablecode for support or download.
7+
# See https://aboutcode.org for more information about nexB OSS projects.
8+
#
9+
import logging
10+
from datetime import datetime
11+
from datetime import timezone
12+
from traceback import format_exc as traceback_format_exc
13+
from typing import Callable
14+
15+
from django.db import transaction
16+
17+
from vulnerabilities import import_runner
18+
from vulnerabilities.importer import AdvisoryData
19+
from vulnerabilities.improver import MAX_CONFIDENCE
20+
from vulnerabilities.improvers import default
21+
from vulnerabilities.models import Advisory
22+
from vulnerabilities.models import Package
23+
from vulnerabilities.models import PackageRelatedVulnerability
24+
from vulnerabilities.models import VulnerabilityReference
25+
from vulnerabilities.models import VulnerabilityRelatedReference
26+
from vulnerabilities.models import VulnerabilitySeverity
27+
from vulnerabilities.models import Weakness
28+
29+
30+
def insert_advisory(advisory: AdvisoryData, pipeline_name: str, logger: Callable):
31+
try:
32+
obj, created = Advisory.objects.get_or_create(
33+
aliases=advisory.aliases,
34+
summary=advisory.summary,
35+
affected_packages=[pkg.to_dict() for pkg in advisory.affected_packages],
36+
references=[ref.to_dict() for ref in advisory.references],
37+
date_published=advisory.date_published,
38+
weaknesses=advisory.weaknesses,
39+
defaults={
40+
"created_by": pipeline_name,
41+
"date_collected": datetime.now(timezone.utc),
42+
},
43+
url=advisory.url,
44+
)
45+
if created:
46+
return obj
47+
except Exception as e:
48+
logger(
49+
f"Error while processing {advisory!r} with aliases {advisory.aliases!r}: {e!r} \n {traceback_format_exc()}",
50+
level=logging.ERROR,
51+
)
52+
53+
54+
@transaction.atomic
55+
def import_advisory(
56+
advisory: Advisory,
57+
pipeline_name: str,
58+
logger: Callable,
59+
confidence: int = MAX_CONFIDENCE,
60+
):
61+
"""
62+
Create initial Vulnerability Package relationships for the advisory,
63+
including references and severity scores.
64+
65+
Package relationships are established only for resolved (concrete) versions.
66+
"""
67+
68+
advisory_data: AdvisoryData = advisory.to_advisory_data()
69+
logger(f"Importing advisory id: {advisory.id}", level=logging.DEBUG)
70+
71+
affected_purls = []
72+
fixed_purls = []
73+
for affected_package in advisory_data.affected_packages:
74+
package_affected_purls, package_fixed_purls = default.get_exact_purls(
75+
affected_package=affected_package
76+
)
77+
affected_purls.extend(package_affected_purls)
78+
fixed_purls.extend(package_fixed_purls)
79+
80+
vulnerability = import_runner.get_or_create_vulnerability_and_aliases(
81+
vulnerability_id=None,
82+
aliases=advisory_data.aliases,
83+
summary=advisory_data.summary,
84+
advisory=advisory,
85+
)
86+
87+
if not vulnerability:
88+
logger(f"Unable to get vulnerability for advisory: {advisory!r}", level=logging.WARNING)
89+
return
90+
91+
for ref in advisory_data.references:
92+
reference = VulnerabilityReference.objects.get_or_none(
93+
reference_id=ref.reference_id,
94+
url=ref.url,
95+
)
96+
if not reference:
97+
reference = import_runner.create_valid_vulnerability_reference(
98+
reference_id=ref.reference_id,
99+
url=ref.url,
100+
)
101+
if not reference:
102+
continue
103+
104+
VulnerabilityRelatedReference.objects.update_or_create(
105+
reference=reference,
106+
vulnerability=vulnerability,
107+
)
108+
for severity in ref.severities:
109+
try:
110+
published_at = str(severity.published_at) if severity.published_at else None
111+
_, created = VulnerabilitySeverity.objects.update_or_create(
112+
scoring_system=severity.system.identifier,
113+
reference=reference,
114+
defaults={
115+
"value": str(severity.value),
116+
"scoring_elements": str(severity.scoring_elements),
117+
"published_at": published_at,
118+
},
119+
)
120+
except:
121+
logger(
122+
f"Failed to create VulnerabilitySeverity for: {severity} with error:\n{traceback_format_exc()}",
123+
level=logging.ERROR,
124+
)
125+
if not created:
126+
logger(
127+
f"Severity updated for reference {ref!r} to value: {severity.value!r} "
128+
f"and scoring_elements: {severity.scoring_elements!r}",
129+
level=logging.DEBUG,
130+
)
131+
132+
for affected_purl in affected_purls or []:
133+
vulnerable_package, _ = Package.objects.get_or_create_from_purl(purl=affected_purl)
134+
PackageRelatedVulnerability(
135+
vulnerability=vulnerability,
136+
package=vulnerable_package,
137+
created_by=pipeline_name,
138+
confidence=confidence,
139+
fix=False,
140+
).update_or_create(advisory=advisory)
141+
142+
for fixed_purl in fixed_purls:
143+
fixed_package, _ = Package.objects.get_or_create_from_purl(purl=fixed_purl)
144+
PackageRelatedVulnerability(
145+
vulnerability=vulnerability,
146+
package=fixed_package,
147+
created_by=pipeline_name,
148+
confidence=confidence,
149+
fix=True,
150+
).update_or_create(advisory=advisory)
151+
152+
if advisory_data.weaknesses and vulnerability:
153+
for cwe_id in advisory_data.weaknesses:
154+
cwe_obj, _ = Weakness.objects.get_or_create(cwe_id=cwe_id)
155+
cwe_obj.vulnerabilities.add(vulnerability)
156+
cwe_obj.save()
157+
158+
advisory.date_imported = datetime.now(timezone.utc)
159+
advisory.save()

0 commit comments

Comments
 (0)