Skip to content

Commit 1a657ca

Browse files
authored
Merge branch 'main' into advisory_id
2 parents 8555d00 + 3fb3e75 commit 1a657ca

8 files changed

+277
-31
lines changed

requirements.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ dateparser==1.1.1
2727
decorator==5.1.1
2828
defusedxml==0.7.1
2929
distro==1.7.0
30-
Django==4.2.17
30+
Django==4.2.20
3131
django-crispy-forms==2.3
3232
django-environ==0.11.2
3333
django-filter==24.3

vulnerabilities/importers/__init__.py

+17-17
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,25 @@
4444
from vulnerabilities.pipelines import pysec_importer
4545

4646
IMPORTERS_REGISTRY = [
47+
nvd_importer.NVDImporterPipeline,
48+
github_importer.GitHubAPIImporterPipeline,
49+
gitlab_importer.GitLabImporterPipeline,
50+
github_osv.GithubOSVImporter,
51+
pypa_importer.PyPaImporterPipeline,
52+
npm_importer.NpmImporterPipeline,
53+
nginx_importer.NginxImporterPipeline,
54+
pysec_importer.PyPIImporterPipeline,
55+
apache_tomcat.ApacheTomcatImporter,
56+
postgresql.PostgreSQLImporter,
57+
debian.DebianImporter,
58+
curl.CurlImporter,
59+
epss.EPSSImporter,
60+
vulnrichment.VulnrichImporter,
61+
alpine_linux_importer.AlpineLinuxImporterPipeline,
62+
ruby.RubyImporter,
63+
apache_kafka.ApacheKafkaImporter,
4764
openssl.OpensslImporter,
4865
redhat.RedhatImporter,
49-
debian.DebianImporter,
50-
postgresql.PostgreSQLImporter,
5166
archlinux.ArchlinuxImporter,
5267
ubuntu.UbuntuImporter,
5368
debian_oval.DebianOvalImporter,
@@ -59,25 +74,10 @@
5974
project_kb_msr2019.ProjectKBMSRImporter,
6075
suse_scores.SUSESeverityScoreImporter,
6176
elixir_security.ElixirSecurityImporter,
62-
apache_tomcat.ApacheTomcatImporter,
6377
xen.XenImporter,
6478
ubuntu_usn.UbuntuUSNImporter,
6579
fireeye.FireyeImporter,
66-
apache_kafka.ApacheKafkaImporter,
6780
oss_fuzz.OSSFuzzImporter,
68-
ruby.RubyImporter,
69-
github_osv.GithubOSVImporter,
70-
curl.CurlImporter,
71-
epss.EPSSImporter,
72-
vulnrichment.VulnrichImporter,
73-
pypa_importer.PyPaImporterPipeline,
74-
npm_importer.NpmImporterPipeline,
75-
nginx_importer.NginxImporterPipeline,
76-
gitlab_importer.GitLabImporterPipeline,
77-
github_importer.GitHubAPIImporterPipeline,
78-
nvd_importer.NVDImporterPipeline,
79-
pysec_importer.PyPIImporterPipeline,
80-
alpine_linux_importer.AlpineLinuxImporterPipeline,
8181
]
8282

8383
IMPORTERS_REGISTRY = {

vulnerabilities/importers/osv.py

+8-1
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,14 @@ def get_affected_purl(affected_pkg, raw_id):
220220
f"No PackageURL possible: {purl!r} for affected_pkg {affected_pkg} for OSV id: {raw_id}"
221221
)
222222
return
223-
return PackageURL.from_string(str(purl))
223+
try:
224+
package_url = PackageURL.from_string(str(purl))
225+
return package_url
226+
except:
227+
logger.error(
228+
f"Invalid PackageURL: {purl!r} for affected_pkg {affected_pkg} for OSV id: {raw_id}"
229+
)
230+
return None
224231

225232

226233
def get_affected_version_range(affected_pkg, raw_id, supported_ecosystem):

vulnerabilities/improvers/__init__.py

+2
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
from vulnerabilities.pipelines import enhance_with_kev
1919
from vulnerabilities.pipelines import enhance_with_metasploit
2020
from vulnerabilities.pipelines import flag_ghost_packages
21+
from vulnerabilities.pipelines import populate_vulnerability_summary_pipeline
2122
from vulnerabilities.pipelines import remove_duplicate_advisories
2223

2324
IMPROVERS_REGISTRY = [
@@ -47,6 +48,7 @@
4748
collect_commits.CollectFixCommitsPipeline,
4849
add_cvss31_to_CVEs.CVEAdvisoryMappingPipeline,
4950
remove_duplicate_advisories.RemoveDuplicateAdvisoriesPipeline,
51+
populate_vulnerability_summary_pipeline.PopulateVulnerabilitySummariesPipeline,
5052
]
5153

5254
IMPROVERS_REGISTRY = {

vulnerabilities/pipelines/alpine_linux_importer.py

+16-10
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,8 @@ def load_advisories(
202202
level=logging.DEBUG,
203203
)
204204
continue
205-
205+
# fixed_vulns is a list of strings and each string is a space-separated
206+
# list of aliases and CVES
206207
for vuln_ids in fixed_vulns:
207208
if not isinstance(vuln_ids, str):
208209
if logger:
@@ -211,15 +212,16 @@ def load_advisories(
211212
level=logging.DEBUG,
212213
)
213214
continue
214-
vuln_ids = vuln_ids.split()
215-
aliases = []
216-
vuln_id = vuln_ids[0]
217-
# check for valid vuln ID, if there is valid vuln ID then iterate over
218-
# the remaining elements of the list else iterate over the whole list
219-
# and also check if the initial element is a reference or not
220-
if is_cve(vuln_id):
221-
aliases = [vuln_id]
222-
vuln_ids = vuln_ids[1:]
215+
vuln_ids = vuln_ids.strip().split()
216+
if not vuln_ids:
217+
if logger:
218+
logger(
219+
f"{vuln_ids!r} is empty",
220+
level=logging.DEBUG,
221+
)
222+
continue
223+
aliases = vuln_ids
224+
223225
references = []
224226
for reference_id in vuln_ids:
225227

@@ -232,6 +234,10 @@ def load_advisories(
232234
elif reference_id.startswith("wnpa-sec"):
233235
references.append(WireSharkReference.from_id(wnpa_sec_id=reference_id))
234236

237+
elif not reference_id.startswith("CVE"):
238+
if logger:
239+
logger(f"Unknown reference id {reference_id!r}", level=logging.DEBUG)
240+
235241
qualifiers = {
236242
"distroversion": distroversion,
237243
"reponame": reponame,
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#
2+
# Copyright (c) nexB Inc. and others. All rights reserved.
3+
# VulnerableCode is a trademark of nexB Inc.
4+
# SPDX-License-Identifier: Apache-2.0
5+
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
6+
# See https://github.com/aboutcode-org/vulnerablecode for support or download.
7+
# See https://aboutcode.org for more information about nexB OSS projects.
8+
#
9+
10+
import logging
11+
12+
from aboutcode.pipeline import LoopProgress
13+
from django.db.models import Q
14+
15+
from vulnerabilities.models import Advisory
16+
from vulnerabilities.models import Vulnerability
17+
from vulnerabilities.pipelines import VulnerableCodePipeline
18+
19+
20+
class PopulateVulnerabilitySummariesPipeline(VulnerableCodePipeline):
21+
"""Pipeline to populate missing vulnerability summaries from advisories."""
22+
23+
pipeline_id = "populate_vulnerability_summaries"
24+
25+
@classmethod
26+
def steps(cls):
27+
return (cls.populate_missing_summaries,)
28+
29+
def populate_missing_summaries(self):
30+
"""Find vulnerabilities with missing summaries and populate them using advisories with the same aliases."""
31+
vulnerabilities_qs = Vulnerability.objects.filter(summary="")
32+
self.log(
33+
f"Processing {vulnerabilities_qs.count()} vulnerabilities without summaries",
34+
level=logging.INFO,
35+
)
36+
37+
progress = LoopProgress(total_iterations=vulnerabilities_qs.count(), logger=self.log)
38+
39+
vulnerabilities_to_be_updated = []
40+
41+
for vulnerability in progress.iter(vulnerabilities_qs.iterator()):
42+
cve_alias = vulnerability.aliases.filter(alias__startswith="CVE-").first()
43+
44+
if not cve_alias:
45+
self.log(
46+
f"Vulnerability {vulnerability.vulnerability_id} has no CVE alias",
47+
level=logging.DEBUG,
48+
)
49+
continue
50+
51+
matching_advisories = Advisory.objects.filter(
52+
aliases=cve_alias, created_by="nvd_importer"
53+
).exclude(summary="")
54+
55+
if matching_advisories.exists():
56+
best_advisory = matching_advisories.order_by("-date_collected").first()
57+
# Note: we filtered above to only get non-empty summaries
58+
vulnerability.summary = best_advisory.summary
59+
vulnerabilities_to_be_updated.append(vulnerability)
60+
self.log(
61+
f"Updated summary for vulnerability {vulnerability.vulnerability_id}",
62+
level=logging.INFO,
63+
)
64+
else:
65+
self.log(f"No advisory found for alias {cve_alias}", level=logging.DEBUG)
66+
Vulnerability.objects.bulk_update(vulnerabilities_to_be_updated, ["summary"])
67+
self.log(
68+
f"Successfully populated {len(vulnerabilities_to_be_updated)} vulnerabilities with summary",
69+
level=logging.INFO,
70+
)
71+
self.log("Pipeline completed", level=logging.INFO)

vulnerabilities/tests/pipelines/test_alpine_linux_importer_pipeline.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ def test_process_record():
3131
logger = TestLogger()
3232
expected_advisories = [
3333
AdvisoryData(
34-
aliases=[],
34+
aliases=["XSA-248"],
3535
summary="",
3636
affected_packages=[
3737
AffectedPackage(
@@ -138,7 +138,7 @@ def test_process_record():
138138
url="https://secdb.alpinelinux.org/v3.11/",
139139
),
140140
AdvisoryData(
141-
aliases=["CVE-2018-7540"],
141+
aliases=["CVE-2018-7540", "XSA-252"],
142142
summary="",
143143
affected_packages=[
144144
AffectedPackage(
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
#
2+
# Copyright (c) nexB Inc. and others. All rights reserved.
3+
# VulnerableCode is a trademark of nexB Inc.
4+
# SPDX-License-Identifier: Apache-2.0
5+
# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
6+
# See https://github.com/aboutcode-org/vulnerablecode for support or download.
7+
# See https://aboutcode.org for more information about nexB OSS projects.
8+
#
9+
10+
import datetime
11+
from pathlib import Path
12+
13+
import pytz
14+
from django.test import TestCase
15+
16+
from vulnerabilities.models import Advisory
17+
from vulnerabilities.models import Alias
18+
from vulnerabilities.models import Vulnerability
19+
from vulnerabilities.pipelines.populate_vulnerability_summary_pipeline import (
20+
PopulateVulnerabilitySummariesPipeline,
21+
)
22+
23+
24+
class PopulateVulnerabilitySummariesPipelineTest(TestCase):
25+
def setUp(self):
26+
self.data = Path(__file__).parent.parent / "test_data"
27+
28+
def test_populate_missing_summaries_from_nvd(self):
29+
"""
30+
Test that vulnerabilities without summaries get them from NVD advisories.
31+
"""
32+
33+
# Create a vulnerability without a summary
34+
vulnerability = Vulnerability.objects.create(
35+
vulnerability_id="VCID-1234",
36+
summary="",
37+
)
38+
alias = Alias.objects.create(alias="CVE-2024-1234", vulnerability=vulnerability)
39+
40+
# Create an NVD advisory with a summary
41+
adv = Advisory.objects.create(
42+
summary="Test vulnerability summary",
43+
created_by="nvd_importer",
44+
date_collected=datetime.datetime(2024, 1, 1, tzinfo=pytz.UTC),
45+
unique_content_id="Test",
46+
)
47+
adv.aliases.add(alias)
48+
49+
# Run the pipeline
50+
pipeline = PopulateVulnerabilitySummariesPipeline()
51+
pipeline.populate_missing_summaries()
52+
53+
# Check that the vulnerability now has a summary
54+
vulnerability.refresh_from_db()
55+
self.assertEqual(vulnerability.summary, "Test vulnerability summary")
56+
57+
def test_no_matching_advisory(self):
58+
"""
59+
Test handling of vulnerabilities that have no matching NVD advisory.
60+
"""
61+
# Create a vulnerability without a summary
62+
vulnerability = Vulnerability.objects.create(
63+
vulnerability_id="VCID-1234",
64+
summary="",
65+
)
66+
Alias.objects.create(alias="CVE-2024-1234", vulnerability=vulnerability)
67+
68+
# Run the pipeline
69+
pipeline = PopulateVulnerabilitySummariesPipeline()
70+
pipeline.populate_missing_summaries()
71+
72+
# Check that the vulnerability still has no summary
73+
vulnerability.refresh_from_db()
74+
self.assertEqual(vulnerability.summary, "")
75+
76+
def test_vulnerability_without_alias(self):
77+
"""
78+
Test handling of vulnerabilities that have no aliases.
79+
"""
80+
81+
# Create a vulnerability without a summary or alias
82+
vulnerability = Vulnerability.objects.create(
83+
vulnerability_id="VCID-1234",
84+
summary="",
85+
)
86+
87+
# Run the pipeline
88+
pipeline = PopulateVulnerabilitySummariesPipeline()
89+
pipeline.populate_missing_summaries()
90+
91+
# Check that the vulnerability still has no summary
92+
vulnerability.refresh_from_db()
93+
self.assertEqual(vulnerability.summary, "")
94+
95+
def test_non_nvd_advisory_ignored(self):
96+
"""
97+
Test that advisories from sources other than NVD are ignored.
98+
"""
99+
100+
# Create a vulnerability without a summary
101+
vulnerability = Vulnerability.objects.create(
102+
vulnerability_id="VCID-1234",
103+
summary="",
104+
)
105+
alias = Alias.objects.create(alias="CVE-2024-1234", vulnerability=vulnerability)
106+
107+
# Create a non-NVD advisory with a summary
108+
adv = Advisory.objects.create(
109+
summary="Test vulnerability summary",
110+
created_by="other_importer",
111+
date_collected=datetime.datetime(2024, 1, 1, tzinfo=pytz.UTC),
112+
unique_content_id="Test",
113+
)
114+
115+
adv.aliases.add(alias)
116+
117+
# Run the pipeline
118+
pipeline = PopulateVulnerabilitySummariesPipeline()
119+
pipeline.populate_missing_summaries()
120+
121+
# Check that the vulnerability still has no summary
122+
vulnerability.refresh_from_db()
123+
self.assertEqual(vulnerability.summary, "")
124+
125+
def test_multiple_matching_advisories(self):
126+
"""
127+
Test that the most recent matching advisory is used when there are multiple.
128+
"""
129+
vulnerability = Vulnerability.objects.create(
130+
vulnerability_id="VCID-1234",
131+
summary="",
132+
)
133+
alias = Alias.objects.create(alias="CVE-2024-1234", vulnerability=vulnerability)
134+
135+
# Create two NVD advisories with the same alias
136+
adv1 = Advisory.objects.create(
137+
summary="First matching advisory",
138+
created_by="nvd_importer",
139+
date_collected=datetime.datetime(2024, 1, 1, tzinfo=pytz.UTC),
140+
unique_content_id="Test",
141+
)
142+
143+
adv1.aliases.add(alias)
144+
145+
adv2 = Advisory.objects.create(
146+
summary="Second matching advisory",
147+
created_by="nvd_importer",
148+
date_collected=datetime.datetime(2024, 1, 2, tzinfo=pytz.UTC),
149+
unique_content_id="Test-1",
150+
)
151+
152+
adv2.aliases.add(alias)
153+
154+
# Run the pipeline
155+
pipeline = PopulateVulnerabilitySummariesPipeline()
156+
pipeline.populate_missing_summaries()
157+
158+
# Check that the vulnerability now has the most recent summary
159+
vulnerability.refresh_from_db()
160+
self.assertEqual(vulnerability.summary, "Second matching advisory")

0 commit comments

Comments
 (0)