From 3aaa7c015982384455ddbced6177d58ad85cd8bb Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 4 Apr 2022 23:41:00 +0530 Subject: [PATCH 1/5] Migrate nvd importer to importer-improver model Signed-off-by: Tushar Goel --- vulnerabilities/importers/__init__.py | 8 +- vulnerabilities/importers/nvd.py | 275 +++++++------- vulnerabilities/improvers/__init__.py | 1 + vulnerabilities/tests/conftest.py | 1 - .../tests/test_data/nvd/nvd-expected.json | 335 ++++++++++++++++++ .../tests/test_data/nvd/nvd_test.json | 285 ++++++++++++++- vulnerabilities/tests/test_nvd.py | 149 +------- 7 files changed, 792 insertions(+), 262 deletions(-) create mode 100644 vulnerabilities/tests/test_data/nvd/nvd-expected.json diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index b9ccaa512..261106685 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -22,7 +22,13 @@ from vulnerabilities.importers import alpine_linux from vulnerabilities.importers import github from vulnerabilities.importers import nginx +from vulnerabilities.importers import nvd -IMPORTERS_REGISTRY = [nginx.NginxImporter, alpine_linux.AlpineImporter, github.GitHubAPIImporter] +IMPORTERS_REGISTRY = [ + nginx.NginxImporter, + alpine_linux.AlpineImporter, + github.GitHubAPIImporter, + nvd.NVDImporter, +] IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY} diff --git a/vulnerabilities/importers/nvd.py b/vulnerabilities/importers/nvd.py index c5e662416..b88e63445 100644 --- a/vulnerabilities/importers/nvd.py +++ b/vulnerabilities/importers/nvd.py @@ -20,150 +20,175 @@ # VulnerableCode is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. -import dataclasses import gzip import json from datetime import date +from typing import Iterable import requests from dateutil import parser as dateparser +from django.db.models.query import QuerySet -from vulnerabilities.helpers import create_etag -from vulnerabilities.importer import Advisory +from vulnerabilities.helpers import get_item +from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.severity_systems import scoring_systems - -BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-{}.json.gz" +from vulnerabilities.improver import Improver +from vulnerabilities.improver import Inference +from vulnerabilities.models import Advisory +from vulnerabilities.severity_systems import SCORING_SYSTEMS class NVDImporter(Importer): - def updated_advisories(self): + spdx_license_expression = "TBD" + + def advisory_data(self): + advisory_data = [] current_year = date.today().year # NVD json feeds start from 2002. for year in range(2002, current_year + 1): - download_url = BASE_URL.format(year) - # Etags are like hashes of web responses. We maintain - # (url, etag) mappings in the DB. `create_etag` creates - # (url, etag) pair. If a (url, etag) already exists then the code - # skips processing the response further to avoid duplicate work - if create_etag(data_src=self, url=download_url, etag_key="etag"): - data = self.fetch(download_url) - yield self.to_advisories(data) - - @staticmethod - def fetch(url): - gz_file = requests.get(url) - data = gzip.decompress(gz_file.content) - return json.loads(data) - - def to_advisories(self, nvd_data): - for cve_item in nvd_data["CVE_Items"]: - if self.is_outdated(cve_item): - continue - - if self.related_to_hardware(cve_item): - continue - - cve_id = cve_item["cve"]["CVE_data_meta"]["ID"] - ref_urls = self.extract_reference_urls(cve_item) - references = [Reference(url=url) for url in ref_urls] - severity_scores = self.extract_severity_scores(cve_item) + download_url = f"https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-{year}.json.gz" + data = fetch(download_url) + advisory_data.extend(to_advisories(data)) + return advisory_data + + +# Isolating network calls for simplicity of testing +def fetch(url): + gz_file = requests.get(url) + data = gzip.decompress(gz_file.content) + return json.loads(data) + + +def extract_summary(cve_item): + """ + Return a summary for a given CVE item. + """ + # In 99% of cases len(cve_item['cve']['description']['description_data']) == 1 , so + # this usually returns cve_item['cve']['description']['description_data'][0]['value'] + # In the remaining 1% cases this returns the longest summary. + summaries = [] + for desc in get_item(cve_item, "cve", "description", "description_data") or []: + if desc.get("value"): + summaries.append(desc["value"]) + return max(summaries, key=len) if summaries else None + + +def to_advisories(nvd_data): + """ + Yield AdvisoryData objects from a NVD json feed. + """ + for cve_item in nvd_data.get("CVE_Items") or []: + cpes = extract_cpes(cve_item) + if related_to_hardware(cpes): + continue + + aliases = [] + cve_id = get_item(cve_item, "cve", "CVE_data_meta", "ID") + ref_urls = extract_reference_urls(cve_item) + references = [] + severity_scores = list(extract_severity_scores(cve_item)) + for cpe in cpes: references.append( Reference( - url=f"https://nvd.nist.gov/vuln/detail/{cve_id}", - reference_id=cve_id, - severities=severity_scores, - ) - ) - summary = self.extract_summary(cve_item) - yield Advisory( - vulnerability_id=cve_id, - summary=summary, - references=references, - ) - - @staticmethod - def extract_summary(cve_item): - # In 99% of cases len(cve_item['cve']['description']['description_data']) == 1 , so - # this usually returns cve_item['cve']['description']['description_data'][0]['value'] - # In the remaining 1% cases this returns the longest summary. - summaries = [desc["value"] for desc in cve_item["cve"]["description"]["description_data"]] - return max(summaries, key=len) - - @staticmethod - def extract_severity_scores(cve_item): - severity_scores = [] - - if cve_item["impact"].get("baseMetricV3"): - severity_scores.append( - VulnerabilitySeverity( - system=scoring_systems["cvssv3"], - value=str(cve_item["impact"]["baseMetricV3"]["cvssV3"]["baseScore"]), + reference_id=cpe, ) ) - severity_scores.append( - VulnerabilitySeverity( - system=scoring_systems["cvssv3_vector"], - value=str(cve_item["impact"]["baseMetricV3"]["cvssV3"]["vectorString"]), - ) - ) - - if cve_item["impact"].get("baseMetricV2"): - severity_scores.append( - VulnerabilitySeverity( - system=scoring_systems["cvssv2"], - value=str(cve_item["impact"]["baseMetricV2"]["cvssV2"]["baseScore"]), - ) - ) - severity_scores.append( - VulnerabilitySeverity( - system=scoring_systems["cvssv2_vector"], - value=str(cve_item["impact"]["baseMetricV2"]["cvssV2"]["vectorString"]), - ) + references.append( + Reference( + url=f"https://nvd.nist.gov/vuln/detail/{cve_id}", + reference_id=cve_id, + severities=severity_scores, ) - - return severity_scores - - def extract_reference_urls(self, cve_item): - urls = set() - for reference in cve_item["cve"]["references"]["reference_data"]: - ref_url = reference["url"] - - if not ref_url: - continue - - if ref_url.startswith("http") or ref_url.startswith("ftp"): - urls.add(ref_url) - - return urls - - def is_outdated(self, cve_item): - cve_last_modified_date = cve_item["lastModifiedDate"] - cve_last_modified_date_obj = dateparser.parse(cve_last_modified_date) - - if self.config.cutoff_date: - return cve_last_modified_date_obj < self.config.cutoff_date - - if self.config.last_run_date: - return cve_last_modified_date_obj < self.config.last_run_date - - return False - - def related_to_hardware(self, cve_item): - for cpe in self.extract_cpes(cve_item): - cpe_comps = cpe.split(":") - # CPE follow the format cpe:cpe_version:product_type:vendor:product - if cpe_comps[2] == "h": - return True - - return False - - @staticmethod - def extract_cpes(cve_item): - cpes = set() - for node in cve_item["configurations"]["nodes"]: - for cpe_data in node.get("cpe_match", []): + ) + if "https://nvd.nist.gov/vuln/detail/{cve_id}" in ref_urls: + ref_urls.remove(f"https://nvd.nist.gov/vuln/detail/{cve_id}") + references.extend([Reference(url=url) for url in ref_urls]) + if cve_id: + aliases.append(cve_id) + summary = extract_summary(cve_item) + yield AdvisoryData( + aliases=aliases, + summary=summary, + references=references, + date_published=dateparser.parse(cve_item.get("publishedDate")), + ) + + +def extract_reference_urls(cve_item): + """ + Return a list of reference URLs for a given CVE item. + """ + urls = set() + for reference in get_item(cve_item, "cve", "references", "reference_data") or []: + ref_url = reference.get("url") + + if not ref_url: + continue + + if ref_url.startswith("http") or ref_url.startswith("ftp"): + urls.add(ref_url) + + return urls + + +def related_to_hardware(cpes): + """ + Return True if the CVE item is related to hardware. + """ + for cpe in cpes: + cpe_comps = cpe.split(":") + # CPE follow the format cpe:cpe_version:product_type:vendor:product + if len(cpe_comps) > 2 and cpe_comps[2] == "h": + return True + + return False + + +def extract_cpes(cve_item): + """ + Return a list of CPEs for a given CVE item. + """ + cpes = set() + for node in get_item(cve_item, "configurations", "nodes") or []: + for cpe_data in node.get("cpe_match") or []: + if cpe_data.get("cpe23Uri"): cpes.add(cpe_data["cpe23Uri"]) - return cpes + return cpes + + +def extract_severity_scores(cve_item): + """ + Yield a vulnerability severity for each `cve_item`. + """ + if get_item(cve_item, "impact", "baseMetricV3"): + yield VulnerabilitySeverity( + system=SCORING_SYSTEMS["cvssv3"], + value=str(get_item(cve_item, "impact", "baseMetricV3", "cvssV3", "baseScore")), + ) + yield VulnerabilitySeverity( + system=SCORING_SYSTEMS["cvssv3_vector"], + value=str(get_item(cve_item, "impact", "baseMetricV3", "cvssV3", "vectorString")), + ) + + if get_item(cve_item, "impact", "baseMetricV2"): + yield VulnerabilitySeverity( + system=SCORING_SYSTEMS["cvssv2"], + value=str(get_item(cve_item, "impact", "baseMetricV2", "cvssV2", "baseScore")), + ) + yield VulnerabilitySeverity( + system=SCORING_SYSTEMS["cvssv2_vector"], + value=str(get_item(cve_item, "impact", "baseMetricV2", "cvssV2", "vectorString")), + ) + + +class NVDBasicImprover(Improver): + @property + def interesting_advisories(self) -> QuerySet: + return Advisory.objects.filter(created_by=NVDImporter.qualified_name) + + def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]: + yield Inference.from_advisory_data( + advisory_data=advisory_data, confidence=100, fixed_purl=None + ) diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index ce640c840..0dbb2a424 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -6,6 +6,7 @@ importers.nginx.NginxBasicImprover, importers.alpine_linux.AlpineBasicImprover, importers.github.GitHubBasicImprover, + importers.nvd.NVDBasicImprover, ] IMPROVERS_REGISTRY = {x.qualified_name: x for x in IMPROVERS_REGISTRY} diff --git a/vulnerabilities/tests/conftest.py b/vulnerabilities/tests/conftest.py index 58ad04b38..0bb246db3 100644 --- a/vulnerabilities/tests/conftest.py +++ b/vulnerabilities/tests/conftest.py @@ -47,7 +47,6 @@ def no_rmtree(monkeypatch): "test_apache_httpd.py", "test_npm.py", "test_apache_kafka.py", - "test_nvd.py", "test_apache_tomcat.py", "test_openssl.py", "test_api.py", diff --git a/vulnerabilities/tests/test_data/nvd/nvd-expected.json b/vulnerabilities/tests/test_data/nvd/nvd-expected.json new file mode 100644 index 000000000..3a848902e --- /dev/null +++ b/vulnerabilities/tests/test_data/nvd/nvd-expected.json @@ -0,0 +1,335 @@ +[ + { + "aliases": [ + "CVE-2005-4895" + ], + "summary": "Multiple integer overflows in TCMalloc (tcmalloc.cc) in gperftools before 0.4 make it easier for context-dependent attackers to perform memory-related attacks such as buffer overflows via a large size value, which causes less memory to be allocated than expected.", + "affected_packages": [], + "references": [ + { + "reference_id": "cpe:2.3:a:csilvers:gperftools:*:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:a:csilvers:gperftools:0.2:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:a:csilvers:gperftools:0.1:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "CVE-2005-4895", + "url": "https://nvd.nist.gov/vuln/detail/CVE-2005-4895", + "severities": [ + { + "system": "cvssv2", + "value": "5.0" + }, + { + "system": "cvssv2_vector", + "value": "AV:N/AC:L/Au:N/C:N/I:N/A:P" + } + ] + }, + { + "reference_id": "", + "url": "http://code.google.com/p/gperftools/source/browse/tags/perftools-0.4/ChangeLog", + "severities": [] + }, + { + "reference_id": "", + "url": "http://kqueue.org/blog/2012/03/05/memory-allocator-security-revisited/", + "severities": [] + } + ], + "date_published": "2012-07-25T19:55:00+00:00" + }, + { + "aliases": [ + "CVE-2003-0001" + ], + "summary": "Multiple ethernet Network Interface Card (NIC) device drivers do not pad frames with null bytes, which allows remote attackers to obtain information from previous packets or kernel memory by using malformed packets, as demonstrated by Etherleak.", + "affected_packages": [], + "references": [ + { + "reference_id": "cpe:2.3:o:freebsd:freebsd:4.3:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:microsoft:windows_2000:*:sp2:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.1:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:freebsd:freebsd:4.7:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.20:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:netbsd:netbsd:1.5.1:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:freebsd:freebsd:4.4:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.15:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:microsoft:windows_2000_terminal_services:*:sp1:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.13:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.5:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:freebsd:freebsd:4.6:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:netbsd:netbsd:1.5:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.12:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.10:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.6:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:freebsd:freebsd:4.5:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.2:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.4:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.11:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:netbsd:netbsd:1.5.2:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:microsoft:windows_2000:*:sp1:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.3:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.14:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.7:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.9:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:microsoft:windows_2000_terminal_services:*:sp2:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.17:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.16:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:freebsd:freebsd:4.2:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:microsoft:windows_2000:*:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.8:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:netbsd:netbsd:1.5.3:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:microsoft:windows_2000_terminal_services:*:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.19:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.18:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:netbsd:netbsd:1.6:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "CVE-2003-0001", + "url": "https://nvd.nist.gov/vuln/detail/CVE-2003-0001", + "severities": [ + { + "system": "cvssv2", + "value": "5.0" + }, + { + "system": "cvssv2_vector", + "value": "AV:N/AC:L/Au:N/C:P/I:N/A:N" + } + ] + }, + { + "reference_id": "", + "url": "http://www.atstake.com/research/advisories/2003/atstake_etherleak_report.pdf", + "severities": [] + }, + { + "reference_id": "", + "url": "http://www.oracle.com/technetwork/topics/security/cpujan2015-1972971.html", + "severities": [] + }, + { + "reference_id": "", + "url": "https://oval.cisecurity.org/repository/search/definition/oval%3Aorg.mitre.oval%3Adef%3A2665", + "severities": [] + }, + { + "reference_id": "", + "url": "http://archives.neohapsis.com/archives/vulnwatch/2003-q1/0016.html", + "severities": [] + }, + { + "reference_id": "", + "url": "http://www.atstake.com/research/advisories/2003/a010603-1.txt", + "severities": [] + }, + { + "reference_id": "", + "url": "http://www.securitytracker.com/id/1040185", + "severities": [] + }, + { + "reference_id": "", + "url": "http://www.securityfocus.com/archive/1/305335/30/26420/threaded", + "severities": [] + }, + { + "reference_id": "", + "url": "http://www.redhat.com/support/errata/RHSA-2003-025.html", + "severities": [] + }, + { + "reference_id": "", + "url": "http://www.redhat.com/support/errata/RHSA-2003-088.html", + "severities": [] + }, + { + "reference_id": "", + "url": "http://www.kb.cert.org/vuls/id/412115", + "severities": [] + }, + { + "reference_id": "", + "url": "http://www.securitytracker.com/id/1031583", + "severities": [] + }, + { + "reference_id": "", + "url": "http://www.osvdb.org/9962", + "severities": [] + }, + { + "reference_id": "", + "url": "http://marc.info/?l=bugtraq&m=104222046632243&w=2", + "severities": [] + }, + { + "reference_id": "", + "url": "http://secunia.com/advisories/7996", + "severities": [] + }, + { + "reference_id": "", + "url": "http://www.securityfocus.com/archive/1/307564/30/26270/threaded", + "severities": [] + } + ], + "date_published": "2003-01-17T05:00:00+00:00" + } +] \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/nvd/nvd_test.json b/vulnerabilities/tests/test_data/nvd/nvd_test.json index a91a8fbf4..5bb675aa5 100644 --- a/vulnerabilities/tests/test_data/nvd/nvd_test.json +++ b/vulnerabilities/tests/test_data/nvd/nvd_test.json @@ -264,6 +264,289 @@ }, "publishedDate": "2016-10-14T16:59Z", "lastModifiedDate": "2018-05-30T01:29Z" - } + }, + { + "cve" : { + "data_type" : "CVE", + "data_format" : "MITRE", + "data_version" : "4.0", + "CVE_data_meta" : { + "ID" : "CVE-2003-0001", + "ASSIGNER" : "cve@mitre.org" + }, + "problemtype" : { + "problemtype_data" : [ { + "description" : [ { + "lang" : "en", + "value" : "CWE-200" + } ] + } ] + }, + "references" : { + "reference_data" : [ { + "url" : "http://www.atstake.com/research/advisories/2003/a010603-1.txt", + "name" : "A010603-1", + "refsource" : "ATSTAKE", + "tags" : [ "Vendor Advisory" ] + }, { + "url" : "http://www.kb.cert.org/vuls/id/412115", + "name" : "VU#412115", + "refsource" : "CERT-VN", + "tags" : [ "Third Party Advisory", "US Government Resource" ] + }, { + "url" : "http://archives.neohapsis.com/archives/vulnwatch/2003-q1/0016.html", + "name" : "20030110 More information regarding Etherleak", + "refsource" : "VULNWATCH", + "tags" : [ ] + }, { + "url" : "http://www.atstake.com/research/advisories/2003/atstake_etherleak_report.pdf", + "name" : "http://www.atstake.com/research/advisories/2003/atstake_etherleak_report.pdf", + "refsource" : "MISC", + "tags" : [ ] + }, { + "url" : "http://www.redhat.com/support/errata/RHSA-2003-025.html", + "name" : "RHSA-2003:025", + "refsource" : "REDHAT", + "tags" : [ ] + }, { + "url" : "http://www.redhat.com/support/errata/RHSA-2003-088.html", + "name" : "RHSA-2003:088", + "refsource" : "REDHAT", + "tags" : [ ] + }, { + "url" : "http://www.osvdb.org/9962", + "name" : "9962", + "refsource" : "OSVDB", + "tags" : [ ] + }, { + "url" : "http://secunia.com/advisories/7996", + "name" : "7996", + "refsource" : "SECUNIA", + "tags" : [ ] + }, { + "url" : "http://www.oracle.com/technetwork/topics/security/cpujan2015-1972971.html", + "name" : "http://www.oracle.com/technetwork/topics/security/cpujan2015-1972971.html", + "refsource" : "CONFIRM", + "tags" : [ ] + }, { + "url" : "http://marc.info/?l=bugtraq&m=104222046632243&w=2", + "name" : "20030110 More information regarding Etherleak", + "refsource" : "BUGTRAQ", + "tags" : [ ] + }, { + "url" : "http://www.securitytracker.com/id/1031583", + "name" : "1031583", + "refsource" : "SECTRACK", + "tags" : [ ] + }, { + "url" : "https://oval.cisecurity.org/repository/search/definition/oval%3Aorg.mitre.oval%3Adef%3A2665", + "name" : "oval:org.mitre.oval:def:2665", + "refsource" : "OVAL", + "tags" : [ ] + }, { + "url" : "http://www.securitytracker.com/id/1040185", + "name" : "1040185", + "refsource" : "SECTRACK", + "tags" : [ ] + }, { + "url" : "http://www.securityfocus.com/archive/1/307564/30/26270/threaded", + "name" : "20030117 Re: More information regarding Etherleak", + "refsource" : "BUGTRAQ", + "tags" : [ ] + }, { + "url" : "http://www.securityfocus.com/archive/1/305335/30/26420/threaded", + "name" : "20030106 Etherleak: Ethernet frame padding information leakage (A010603-1)", + "refsource" : "BUGTRAQ", + "tags" : [ ] + } ] + }, + "description" : { + "description_data" : [ { + "lang" : "en", + "value" : "Multiple ethernet Network Interface Card (NIC) device drivers do not pad frames with null bytes, which allows remote attackers to obtain information from previous packets or kernel memory by using malformed packets, as demonstrated by Etherleak." + } ] + } + }, + "configurations" : { + "CVE_data_version" : "4.0", + "nodes" : [ { + "operator" : "OR", + "children" : [ ], + "cpe_match" : [ { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.1:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:freebsd:freebsd:4.6:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:freebsd:freebsd:4.7:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.15:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.16:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.4:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.5:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.6:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:microsoft:windows_2000:*:sp1:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:microsoft:windows_2000_terminal_services:*:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:netbsd:netbsd:1.6:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:freebsd:freebsd:4.2:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:freebsd:freebsd:4.3:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.11:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.12:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.19:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.2:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.9:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:microsoft:windows_2000:*:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:microsoft:windows_2000:*:sp2:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:netbsd:netbsd:1.5:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:netbsd:netbsd:1.5.1:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.10:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.17:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.18:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.7:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.8:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:microsoft:windows_2000_terminal_services:*:sp1:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:microsoft:windows_2000_terminal_services:*:sp2:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:freebsd:freebsd:4.4:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:freebsd:freebsd:4.5:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.13:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.14:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.20:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.3:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:netbsd:netbsd:1.5.2:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:netbsd:netbsd:1.5.3:*:*:*:*:*:*:*", + "cpe_name" : [ ] + } ] + } ] + }, + "impact" : { + "baseMetricV2" : { + "cvssV2" : { + "version" : "2.0", + "vectorString" : "AV:N/AC:L/Au:N/C:P/I:N/A:N", + "accessVector" : "NETWORK", + "accessComplexity" : "LOW", + "authentication" : "NONE", + "confidentialityImpact" : "PARTIAL", + "integrityImpact" : "NONE", + "availabilityImpact" : "NONE", + "baseScore" : 5.0 + }, + "severity" : "MEDIUM", + "exploitabilityScore" : 10.0, + "impactScore" : 2.9, + "obtainAllPrivilege" : false, + "obtainUserPrivilege" : false, + "obtainOtherPrivilege" : false, + "userInteractionRequired" : false + } + }, + "publishedDate" : "2003-01-17T05:00Z", + "lastModifiedDate" : "2019-04-30T14:27Z" + } ] } \ No newline at end of file diff --git a/vulnerabilities/tests/test_nvd.py b/vulnerabilities/tests/test_nvd.py index 2cd7856d7..3395f917a 100644 --- a/vulnerabilities/tests/test_nvd.py +++ b/vulnerabilities/tests/test_nvd.py @@ -22,148 +22,29 @@ import json import os -from unittest import TestCase -from dateutil import parser as dateparser - -from vulnerabilities.importer import Advisory -from vulnerabilities.importer import Reference -from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.importers import NVDImporter -from vulnerabilities.severity_systems import scoring_systems +from vulnerabilities.importers.nvd import to_advisories BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data/nvd/nvd_test.json") -class TestNVDImporter(TestCase): - @classmethod - def setUpClass(cls): - data_source_cfg = {"etags": {}} - cls.data_src = NVDImporter(1, config=data_source_cfg) - with open(TEST_DATA) as f: - cls.nvd_data = json.load(f) - - def test_extract_cpes(self): - expected_cpes = { - "cpe:2.3:a:csilvers:gperftools:0.1:*:*:*:*:*:*:*", - "cpe:2.3:a:csilvers:gperftools:0.2:*:*:*:*:*:*:*", - "cpe:2.3:h:google:chrome:*:*:*:*:*:*:*:*", - "cpe:2.3:a:csilvers:gperftools:*:*:*:*:*:*:*:*", - } - - found_cpes = set() - for cve_item in self.nvd_data["CVE_Items"]: - found_cpes.update(NVDImporter.extract_cpes(cve_item)) - - assert expected_cpes == found_cpes - - def test_related_to_hardware(self): - # Only CVE-2005-4900 is supposed to be a hardware related - # vulnerability. - for cve_item in self.nvd_data["CVE_Items"]: - expected_result = "CVE-2005-4900" == cve_item["cve"]["CVE_data_meta"]["ID"] - assert self.data_src.related_to_hardware(cve_item) == expected_result - - def test_extract_summary_with_single_summary(self): - expected_summary = ( - "Multiple integer overflows in TCMalloc (tcmalloc.cc) in gperftools " - "before 0.4 make it easier for context-dependent attackers to perform memory-related " - "attacks such as buffer overflows via a large size value, which causes less memory to " - "be allocated than expected." - ) - cve_item = self.nvd_data["CVE_Items"][0] - assert len(cve_item["cve"]["description"]["description_data"]) == 1 - found_summary = NVDImporter.extract_summary(cve_item) - assert found_summary == expected_summary - - def test_extract_summary_with_multiple_summary(self): - expected_summary = ( - "SHA-1 is not collision resistant, which makes it easier for context-dependent " - "attackers to conduct spoofing attacks, as demonstrated by attacks on the use of SHA-1" - " in TLS 1.2. NOTE: this CVE exists to provide a common identifier for referencing " - "this SHA-1 issue; the existence of an identifier is not, by itself, a technology " - "recommendation." - ) - cve_item = self.nvd_data["CVE_Items"][1] - assert len(cve_item["cve"]["description"]["description_data"]) > 1 - found_summary = NVDImporter.extract_summary(cve_item) - assert found_summary == expected_summary - - def test_is_outdated(self): - cve_item = self.nvd_data["CVE_Items"][0] - assert self.data_src.is_outdated(cve_item) is False - - self.data_src.config.cutoff_date = dateparser.parse("2019-08-05 13:14:17.733232+05:30") - assert self.data_src.is_outdated(cve_item) - self.data_src.config.cutoff_date = None # cleanup - - assert self.data_src.is_outdated(cve_item) is False - - self.data_src.config.last_run_date = dateparser.parse("2019-08-05 13:14:17.733232+05:30") - assert self.data_src.is_outdated(cve_item) - - self.data_src.config.last_run_date = dateparser.parse("2000-08-05 13:14:17.733232+05:30") - assert self.data_src.is_outdated(cve_item) is False - self.data_src.config.last_run_date = None # cleanup - - def test_extract_reference_urls(self): - cve_item = self.nvd_data["CVE_Items"][1] - expected_urls = { - "http://ia.cr/2007/474", - "http://shattered.io/", - "http://www.cwi.nl/news/2017/cwi-and-google-announce-first-collision-industry-security-standard-sha-1", # nopep8 - "http://www.securityfocus.com/bid/12577", - "https://arstechnica.com/security/2017/02/at-deaths-door-for-years-widely-used-sha1-function-is-now-dead/", # nopep8 - "https://security.googleblog.com/2015/12/an-update-on-sha-1-certificates-in.html", - "https://security.googleblog.com/2017/02/announcing-first-sha1-collision.html", - "https://sites.google.com/site/itstheshappening", - "https://www.schneier.com/blog/archives/2005/02/sha1_broken.html", - "https://www.schneier.com/blog/archives/2005/08/new_cryptanalyt.html", - } +def load_test_data(): + with open(TEST_DATA) as f: + return json.load(f) - found_urls = self.data_src.extract_reference_urls(cve_item) - assert found_urls == expected_urls +def test_nvd_importer_with_hardware(regen=True): + expected_file = os.path.join(BASE_DIR, "test_data/nvd/nvd-expected.json") - def test_to_advisories(self): + result = [data.to_dict() for data in list(to_advisories(load_test_data()))] - expected_advisories = [ - Advisory( - summary=( - "Multiple integer overflows in TCMalloc (tcmalloc.cc) in gperftools " - "before 0.4 make it easier for context-dependent attackers to perform memory-related " # nopep8 - "attacks such as buffer overflows via a large size value, which causes less memory to " # nopep8 - "be allocated than expected." - ), - references=[ - Reference( - url="http://code.google.com/p/gperftools/source/browse/tags/perftools-0.4/ChangeLog", # nopep8 - ), - Reference( - url="http://kqueue.org/blog/2012/03/05/memory-allocator-security-revisited/", # nopep8 - ), - Reference( - url="https://nvd.nist.gov/vuln/detail/CVE-2005-4895", - severities=[ - VulnerabilitySeverity( - system=scoring_systems["cvssv2"], - value="5.0", - ), - VulnerabilitySeverity( - system=scoring_systems["cvssv2_vector"], - value="AV:N/AC:L/Au:N/C:N/I:N/A:P", - ), - ], - reference_id="CVE-2005-4895", - ), - ], - vulnerability_id="CVE-2005-4895", - ) - ] - assert len(self.nvd_data["CVE_Items"]) == 2 + if regen: + with open(expected_file, "w") as f: + json.dump(result, f, indent=2) + expected = result + else: + with open(expected_file) as f: + expected = json.load(f) - found_advisories = list(self.data_src.to_advisories(self.nvd_data)) - found_advisories = list(map(Advisory.normalized, found_advisories)) - expected_advisories = list(map(Advisory.normalized, expected_advisories)) - assert sorted(found_advisories) == sorted(expected_advisories) + assert result == expected From bff3444ffb1a52a16a4021d23d73fe56cf8a336b Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 5 Apr 2022 16:15:05 +0530 Subject: [PATCH 2/5] Add spdx license expression Signed-off-by: Tushar Goel --- vulnerabilities/importers/nvd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnerabilities/importers/nvd.py b/vulnerabilities/importers/nvd.py index b88e63445..419802faa 100644 --- a/vulnerabilities/importers/nvd.py +++ b/vulnerabilities/importers/nvd.py @@ -41,7 +41,7 @@ class NVDImporter(Importer): - spdx_license_expression = "TBD" + spdx_license_expression = "LicenseRef-scancode-unknown" def advisory_data(self): advisory_data = [] From 08beb9fbd94256b27b737ec26ed579e5d7937a2f Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Sat, 9 Apr 2022 01:00:57 +0530 Subject: [PATCH 3/5] Add unit tests and make model changes for storing cpe as reference_id Signed-off-by: Tushar Goel --- vulnerabilities/importers/nvd.py | 33 ++- ...ter_vulnerabilityreference_reference_id.py | 24 ++ vulnerabilities/models.py | 2 +- vulnerabilities/tests/test_cpe_reference.py | 37 +++ .../tests/test_data/nvd/nvd-expected.json | 216 +++++++++--------- vulnerabilities/tests/test_nvd.py | 140 +++++++++++- 6 files changed, 332 insertions(+), 120 deletions(-) create mode 100644 vulnerabilities/migrations/0007_alter_vulnerabilityreference_reference_id.py create mode 100644 vulnerabilities/tests/test_cpe_reference.py diff --git a/vulnerabilities/importers/nvd.py b/vulnerabilities/importers/nvd.py index 419802faa..d66ef29ba 100644 --- a/vulnerabilities/importers/nvd.py +++ b/vulnerabilities/importers/nvd.py @@ -111,7 +111,7 @@ def to_advisories(nvd_data): yield AdvisoryData( aliases=aliases, summary=summary, - references=references, + references=sorted(references), date_published=dateparser.parse(cve_item.get("publishedDate")), ) @@ -127,7 +127,12 @@ def extract_reference_urls(cve_item): if not ref_url: continue - if ref_url.startswith("http") or ref_url.startswith("ftp"): + if ref_url.startswith( + ( + "http", + "ftp", + ) + ): urls.add(ref_url) return urls @@ -153,8 +158,9 @@ def extract_cpes(cve_item): cpes = set() for node in get_item(cve_item, "configurations", "nodes") or []: for cpe_data in node.get("cpe_match") or []: - if cpe_data.get("cpe23Uri"): - cpes.add(cpe_data["cpe23Uri"]) + cpe23_uri = cpe_data.get("cpe23Uri") + if cpe23_uri: + cpes.add(cpe23_uri) return cpes @@ -162,24 +168,31 @@ def extract_severity_scores(cve_item): """ Yield a vulnerability severity for each `cve_item`. """ - if get_item(cve_item, "impact", "baseMetricV3"): + if not isinstance(cve_item, dict): + return None + impact = cve_item.get("impact") or {} + base_metric_v3 = impact.get("baseMetricV3") or {} + if base_metric_v3: + cvss_v3 = get_item(base_metric_v3, "cvssV3") yield VulnerabilitySeverity( system=SCORING_SYSTEMS["cvssv3"], - value=str(get_item(cve_item, "impact", "baseMetricV3", "cvssV3", "baseScore")), + value=str(cvss_v3.get("baseScore") or ""), ) yield VulnerabilitySeverity( system=SCORING_SYSTEMS["cvssv3_vector"], - value=str(get_item(cve_item, "impact", "baseMetricV3", "cvssV3", "vectorString")), + value=str(cvss_v3.get("vectorString") or ""), ) - if get_item(cve_item, "impact", "baseMetricV2"): + base_metric_v2 = impact.get("baseMetricV2") or {} + if base_metric_v2: + cvss_v2 = base_metric_v2.get("cvssV2") or {} yield VulnerabilitySeverity( system=SCORING_SYSTEMS["cvssv2"], - value=str(get_item(cve_item, "impact", "baseMetricV2", "cvssV2", "baseScore")), + value=str(cvss_v2.get("baseScore") or ""), ) yield VulnerabilitySeverity( system=SCORING_SYSTEMS["cvssv2_vector"], - value=str(get_item(cve_item, "impact", "baseMetricV2", "cvssV2", "vectorString")), + value=str(cvss_v2.get("vectorString") or ""), ) diff --git a/vulnerabilities/migrations/0007_alter_vulnerabilityreference_reference_id.py b/vulnerabilities/migrations/0007_alter_vulnerabilityreference_reference_id.py new file mode 100644 index 000000000..14bb61f13 --- /dev/null +++ b/vulnerabilities/migrations/0007_alter_vulnerabilityreference_reference_id.py @@ -0,0 +1,24 @@ +# Generated by Django 4.0.2 on 2022-04-08 18:53 + +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0006_alter_advisory_unique_together"), + ] + + operations = [ + migrations.AlterField( + model_name="vulnerabilityreference", + name="reference_id", + field=models.CharField( + blank=True, + help_text="An optional reference ID, such as DSA-4465-1 when available", + max_length=180, + null=True, + ), + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index e8062fa7c..3b039f7df 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -104,7 +104,7 @@ class VulnerabilityReference(models.Model): max_length=1024, help_text="URL to the vulnerability reference", blank=True ) reference_id = models.CharField( - max_length=50, + max_length=180, help_text="An optional reference ID, such as DSA-4465-1 when available", blank=True, null=True, diff --git a/vulnerabilities/tests/test_cpe_reference.py b/vulnerabilities/tests/test_cpe_reference.py new file mode 100644 index 000000000..3c9440417 --- /dev/null +++ b/vulnerabilities/tests/test_cpe_reference.py @@ -0,0 +1,37 @@ +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. +import pytest + +from vulnerabilities.improve_runner import get_or_create_vulnerability_and_aliases +from vulnerabilities.models import Vulnerability +from vulnerabilities.models import VulnerabilityReference + + +@pytest.mark.django_db +def test_cpe_as_reference_id_in_db(): + vulnerability = Vulnerability(summary="lorem ipsum" * 10) + vulnerability.save() + VulnerabilityReference.objects.get_or_create( + reference_id="cpe:2.3:a:microsoft:windows_10:10.0.17134:*:*:*:*:*:*:*" * 3, + url="https://foo.com", + vulnerability=vulnerability, + ) diff --git a/vulnerabilities/tests/test_data/nvd/nvd-expected.json b/vulnerabilities/tests/test_data/nvd/nvd-expected.json index 3a848902e..39bb178ae 100644 --- a/vulnerabilities/tests/test_data/nvd/nvd-expected.json +++ b/vulnerabilities/tests/test_data/nvd/nvd-expected.json @@ -7,18 +7,13 @@ "affected_packages": [], "references": [ { - "reference_id": "cpe:2.3:a:csilvers:gperftools:*:*:*:*:*:*:*:*", - "url": "", - "severities": [] - }, - { - "reference_id": "cpe:2.3:a:csilvers:gperftools:0.2:*:*:*:*:*:*:*", - "url": "", + "reference_id": "", + "url": "http://code.google.com/p/gperftools/source/browse/tags/perftools-0.4/ChangeLog", "severities": [] }, { - "reference_id": "cpe:2.3:a:csilvers:gperftools:0.1:*:*:*:*:*:*:*", - "url": "", + "reference_id": "", + "url": "http://kqueue.org/blog/2012/03/05/memory-allocator-security-revisited/", "severities": [] }, { @@ -36,13 +31,18 @@ ] }, { - "reference_id": "", - "url": "http://code.google.com/p/gperftools/source/browse/tags/perftools-0.4/ChangeLog", + "reference_id": "cpe:2.3:a:csilvers:gperftools:*:*:*:*:*:*:*:*", + "url": "", "severities": [] }, { - "reference_id": "", - "url": "http://kqueue.org/blog/2012/03/05/memory-allocator-security-revisited/", + "reference_id": "cpe:2.3:a:csilvers:gperftools:0.1:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:a:csilvers:gperftools:0.2:*:*:*:*:*:*:*", + "url": "", "severities": [] } ], @@ -56,142 +56,151 @@ "affected_packages": [], "references": [ { - "reference_id": "cpe:2.3:o:freebsd:freebsd:4.3:*:*:*:*:*:*:*", - "url": "", + "reference_id": "", + "url": "http://archives.neohapsis.com/archives/vulnwatch/2003-q1/0016.html", "severities": [] }, { - "reference_id": "cpe:2.3:o:microsoft:windows_2000:*:sp2:*:*:*:*:*:*", - "url": "", + "reference_id": "", + "url": "http://marc.info/?l=bugtraq&m=104222046632243&w=2", "severities": [] }, { - "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.1:*:*:*:*:*:*:*", - "url": "", + "reference_id": "", + "url": "http://secunia.com/advisories/7996", "severities": [] }, { - "reference_id": "cpe:2.3:o:freebsd:freebsd:4.7:*:*:*:*:*:*:*", - "url": "", + "reference_id": "", + "url": "http://www.atstake.com/research/advisories/2003/a010603-1.txt", "severities": [] }, { - "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.20:*:*:*:*:*:*:*", - "url": "", + "reference_id": "", + "url": "http://www.atstake.com/research/advisories/2003/atstake_etherleak_report.pdf", "severities": [] }, { - "reference_id": "cpe:2.3:o:netbsd:netbsd:1.5.1:*:*:*:*:*:*:*", - "url": "", + "reference_id": "", + "url": "http://www.kb.cert.org/vuls/id/412115", "severities": [] }, { - "reference_id": "cpe:2.3:o:freebsd:freebsd:4.4:*:*:*:*:*:*:*", - "url": "", + "reference_id": "", + "url": "http://www.oracle.com/technetwork/topics/security/cpujan2015-1972971.html", "severities": [] }, { - "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.15:*:*:*:*:*:*:*", - "url": "", + "reference_id": "", + "url": "http://www.osvdb.org/9962", "severities": [] }, { - "reference_id": "cpe:2.3:o:microsoft:windows_2000_terminal_services:*:sp1:*:*:*:*:*:*", - "url": "", + "reference_id": "", + "url": "http://www.redhat.com/support/errata/RHSA-2003-025.html", "severities": [] }, { - "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.13:*:*:*:*:*:*:*", - "url": "", + "reference_id": "", + "url": "http://www.redhat.com/support/errata/RHSA-2003-088.html", "severities": [] }, { - "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.5:*:*:*:*:*:*:*", - "url": "", + "reference_id": "", + "url": "http://www.securityfocus.com/archive/1/305335/30/26420/threaded", "severities": [] }, { - "reference_id": "cpe:2.3:o:freebsd:freebsd:4.6:*:*:*:*:*:*:*", - "url": "", + "reference_id": "", + "url": "http://www.securityfocus.com/archive/1/307564/30/26270/threaded", "severities": [] }, { - "reference_id": "cpe:2.3:o:netbsd:netbsd:1.5:*:*:*:*:*:*:*", - "url": "", + "reference_id": "", + "url": "http://www.securitytracker.com/id/1031583", "severities": [] }, { - "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.12:*:*:*:*:*:*:*", - "url": "", + "reference_id": "", + "url": "http://www.securitytracker.com/id/1040185", "severities": [] }, { - "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.10:*:*:*:*:*:*:*", - "url": "", + "reference_id": "", + "url": "https://oval.cisecurity.org/repository/search/definition/oval%3Aorg.mitre.oval%3Adef%3A2665", "severities": [] }, { - "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.6:*:*:*:*:*:*:*", - "url": "", - "severities": [] + "reference_id": "CVE-2003-0001", + "url": "https://nvd.nist.gov/vuln/detail/CVE-2003-0001", + "severities": [ + { + "system": "cvssv2", + "value": "5.0" + }, + { + "system": "cvssv2_vector", + "value": "AV:N/AC:L/Au:N/C:P/I:N/A:N" + } + ] }, { - "reference_id": "cpe:2.3:o:freebsd:freebsd:4.5:*:*:*:*:*:*:*", + "reference_id": "cpe:2.3:o:freebsd:freebsd:4.2:*:*:*:*:*:*:*", "url": "", "severities": [] }, { - "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.2:*:*:*:*:*:*:*", + "reference_id": "cpe:2.3:o:freebsd:freebsd:4.3:*:*:*:*:*:*:*", "url": "", "severities": [] }, { - "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.4:*:*:*:*:*:*:*", + "reference_id": "cpe:2.3:o:freebsd:freebsd:4.4:*:*:*:*:*:*:*", "url": "", "severities": [] }, { - "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.11:*:*:*:*:*:*:*", + "reference_id": "cpe:2.3:o:freebsd:freebsd:4.5:*:*:*:*:*:*:*", "url": "", "severities": [] }, { - "reference_id": "cpe:2.3:o:netbsd:netbsd:1.5.2:*:*:*:*:*:*:*", + "reference_id": "cpe:2.3:o:freebsd:freebsd:4.6:*:*:*:*:*:*:*", "url": "", "severities": [] }, { - "reference_id": "cpe:2.3:o:microsoft:windows_2000:*:sp1:*:*:*:*:*:*", + "reference_id": "cpe:2.3:o:freebsd:freebsd:4.7:*:*:*:*:*:*:*", "url": "", "severities": [] }, { - "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.3:*:*:*:*:*:*:*", + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.10:*:*:*:*:*:*:*", "url": "", "severities": [] }, { - "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.14:*:*:*:*:*:*:*", + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.11:*:*:*:*:*:*:*", "url": "", "severities": [] }, { - "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.7:*:*:*:*:*:*:*", + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.12:*:*:*:*:*:*:*", "url": "", "severities": [] }, { - "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.9:*:*:*:*:*:*:*", + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.13:*:*:*:*:*:*:*", "url": "", "severities": [] }, { - "reference_id": "cpe:2.3:o:microsoft:windows_2000_terminal_services:*:sp2:*:*:*:*:*:*", + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.14:*:*:*:*:*:*:*", "url": "", "severities": [] }, { - "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.17:*:*:*:*:*:*:*", + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.15:*:*:*:*:*:*:*", "url": "", "severities": [] }, @@ -201,132 +210,123 @@ "severities": [] }, { - "reference_id": "cpe:2.3:o:freebsd:freebsd:4.2:*:*:*:*:*:*:*", + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.17:*:*:*:*:*:*:*", "url": "", "severities": [] }, { - "reference_id": "cpe:2.3:o:microsoft:windows_2000:*:*:*:*:*:*:*:*", + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.18:*:*:*:*:*:*:*", "url": "", "severities": [] }, { - "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.8:*:*:*:*:*:*:*", + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.19:*:*:*:*:*:*:*", "url": "", "severities": [] }, { - "reference_id": "cpe:2.3:o:netbsd:netbsd:1.5.3:*:*:*:*:*:*:*", + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.1:*:*:*:*:*:*:*", "url": "", "severities": [] }, { - "reference_id": "cpe:2.3:o:microsoft:windows_2000_terminal_services:*:*:*:*:*:*:*:*", + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.20:*:*:*:*:*:*:*", "url": "", "severities": [] }, { - "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.19:*:*:*:*:*:*:*", + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.2:*:*:*:*:*:*:*", "url": "", "severities": [] }, { - "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.18:*:*:*:*:*:*:*", + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.3:*:*:*:*:*:*:*", "url": "", "severities": [] }, { - "reference_id": "cpe:2.3:o:netbsd:netbsd:1.6:*:*:*:*:*:*:*", + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.4:*:*:*:*:*:*:*", "url": "", "severities": [] }, { - "reference_id": "CVE-2003-0001", - "url": "https://nvd.nist.gov/vuln/detail/CVE-2003-0001", - "severities": [ - { - "system": "cvssv2", - "value": "5.0" - }, - { - "system": "cvssv2_vector", - "value": "AV:N/AC:L/Au:N/C:P/I:N/A:N" - } - ] + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.5:*:*:*:*:*:*:*", + "url": "", + "severities": [] }, { - "reference_id": "", - "url": "http://www.atstake.com/research/advisories/2003/atstake_etherleak_report.pdf", + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.6:*:*:*:*:*:*:*", + "url": "", "severities": [] }, { - "reference_id": "", - "url": "http://www.oracle.com/technetwork/topics/security/cpujan2015-1972971.html", + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.7:*:*:*:*:*:*:*", + "url": "", "severities": [] }, { - "reference_id": "", - "url": "https://oval.cisecurity.org/repository/search/definition/oval%3Aorg.mitre.oval%3Adef%3A2665", + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.8:*:*:*:*:*:*:*", + "url": "", "severities": [] }, { - "reference_id": "", - "url": "http://archives.neohapsis.com/archives/vulnwatch/2003-q1/0016.html", + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.9:*:*:*:*:*:*:*", + "url": "", "severities": [] }, { - "reference_id": "", - "url": "http://www.atstake.com/research/advisories/2003/a010603-1.txt", + "reference_id": "cpe:2.3:o:microsoft:windows_2000:*:*:*:*:*:*:*:*", + "url": "", "severities": [] }, { - "reference_id": "", - "url": "http://www.securitytracker.com/id/1040185", + "reference_id": "cpe:2.3:o:microsoft:windows_2000:*:sp1:*:*:*:*:*:*", + "url": "", "severities": [] }, { - "reference_id": "", - "url": "http://www.securityfocus.com/archive/1/305335/30/26420/threaded", + "reference_id": "cpe:2.3:o:microsoft:windows_2000:*:sp2:*:*:*:*:*:*", + "url": "", "severities": [] }, { - "reference_id": "", - "url": "http://www.redhat.com/support/errata/RHSA-2003-025.html", + "reference_id": "cpe:2.3:o:microsoft:windows_2000_terminal_services:*:*:*:*:*:*:*:*", + "url": "", "severities": [] }, { - "reference_id": "", - "url": "http://www.redhat.com/support/errata/RHSA-2003-088.html", + "reference_id": "cpe:2.3:o:microsoft:windows_2000_terminal_services:*:sp1:*:*:*:*:*:*", + "url": "", "severities": [] }, { - "reference_id": "", - "url": "http://www.kb.cert.org/vuls/id/412115", + "reference_id": "cpe:2.3:o:microsoft:windows_2000_terminal_services:*:sp2:*:*:*:*:*:*", + "url": "", "severities": [] }, { - "reference_id": "", - "url": "http://www.securitytracker.com/id/1031583", + "reference_id": "cpe:2.3:o:netbsd:netbsd:1.5.1:*:*:*:*:*:*:*", + "url": "", "severities": [] }, { - "reference_id": "", - "url": "http://www.osvdb.org/9962", + "reference_id": "cpe:2.3:o:netbsd:netbsd:1.5.2:*:*:*:*:*:*:*", + "url": "", "severities": [] }, { - "reference_id": "", - "url": "http://marc.info/?l=bugtraq&m=104222046632243&w=2", + "reference_id": "cpe:2.3:o:netbsd:netbsd:1.5.3:*:*:*:*:*:*:*", + "url": "", "severities": [] }, { - "reference_id": "", - "url": "http://secunia.com/advisories/7996", + "reference_id": "cpe:2.3:o:netbsd:netbsd:1.5:*:*:*:*:*:*:*", + "url": "", "severities": [] }, { - "reference_id": "", - "url": "http://www.securityfocus.com/archive/1/307564/30/26270/threaded", + "reference_id": "cpe:2.3:o:netbsd:netbsd:1.6:*:*:*:*:*:*:*", + "url": "", "severities": [] } ], diff --git a/vulnerabilities/tests/test_nvd.py b/vulnerabilities/tests/test_nvd.py index 3395f917a..960dfaef9 100644 --- a/vulnerabilities/tests/test_nvd.py +++ b/vulnerabilities/tests/test_nvd.py @@ -23,6 +23,10 @@ import json import os +from vulnerabilities.importers.nvd import extract_cpes +from vulnerabilities.importers.nvd import extract_reference_urls +from vulnerabilities.importers.nvd import extract_summary +from vulnerabilities.importers.nvd import related_to_hardware from vulnerabilities.importers.nvd import to_advisories BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -34,7 +38,7 @@ def load_test_data(): return json.load(f) -def test_nvd_importer_with_hardware(regen=True): +def test_nvd_importer_with_hardware(regen=False): expected_file = os.path.join(BASE_DIR, "test_data/nvd/nvd-expected.json") result = [data.to_dict() for data in list(to_advisories(load_test_data()))] @@ -48,3 +52,137 @@ def test_nvd_importer_with_hardware(regen=True): expected = json.load(f) assert result == expected + + +def get_cve_item(): + + return { + "cve": { + "data_type": "CVE", + "data_format": "MITRE", + "data_version": "4.0", + "CVE_data_meta": {"ID": "CVE-2005-4895", "ASSIGNER": "cve@mitre.org"}, + "problemtype": { + "problemtype_data": [{"description": [{"lang": "en", "value": "CWE-189"}]}] + }, + "references": { + "reference_data": [ + { + "url": "http://code.google.com/p/gperftools/source/browse/tags/perftools-0.4/ChangeLog", + "name": "http://code.google.com/p/gperftools/source/browse/tags/perftools-0.4/ChangeLog", + "refsource": "CONFIRM", + "tags": [], + }, + { + "url": "http://kqueue.org/blog/2012/03/05/memory-allocator-security-revisited/", + "name": "http://kqueue.org/blog/2012/03/05/memory-allocator-security-revisited/", + "refsource": "MISC", + "tags": [], + }, + ] + }, + "description": { + "description_data": [ + { + "lang": "en", + "value": "Multiple integer overflows in TCMalloc (tcmalloc.cc) in gperftools before 0.4 make it easier for context-dependent attackers to perform memory-related attacks such as buffer overflows via a large size value, which causes less memory to be allocated than expected.", + } + ] + }, + }, + "configurations": { + "CVE_data_version": "4.0", + "nodes": [ + { + "operator": "OR", + "cpe_match": [ + { + "vulnerable": True, + "cpe23Uri": "cpe:2.3:a:csilvers:gperftools:0.1:*:*:*:*:*:*:*", + }, + { + "vulnerable": True, + "cpe23Uri": "cpe:2.3:a:csilvers:gperftools:0.2:*:*:*:*:*:*:*", + }, + { + "vulnerable": True, + "cpe23Uri": "cpe:2.3:a:csilvers:gperftools:*:*:*:*:*:*:*:*", + "versionEndIncluding": "0.3", + }, + ], + } + ], + }, + "impact": { + "baseMetricV2": { + "cvssV2": { + "version": "2.0", + "vectorString": "AV:N/AC:L/Au:N/C:N/I:N/A:P", + "accessVector": "NETWORK", + "accessComplexity": "LOW", + "authentication": "NONE", + "confidentialityImpact": "NONE", + "integrityImpact": "NONE", + "availabilityImpact": "PARTIAL", + "baseScore": 5.0, + }, + "severity": "MEDIUM", + "exploitabilityScore": 10.0, + "impactScore": 2.9, + "obtainAllPrivilege": False, + "obtainUserPrivilege": False, + "obtainOtherPrivilege": False, + "userInteractionRequired": False, + } + }, + "publishedDate": "2012-07-25T19:55Z", + "lastModifiedDate": "2012-08-09T04:00Z", + } + + +def test_extract_cpes(): + expected_cpes = { + "cpe:2.3:a:csilvers:gperftools:0.1:*:*:*:*:*:*:*", + "cpe:2.3:a:csilvers:gperftools:0.2:*:*:*:*:*:*:*", + "cpe:2.3:a:csilvers:gperftools:*:*:*:*:*:*:*:*", + } + + found_cpes = set() + found_cpes.update(extract_cpes(get_cve_item())) + + assert found_cpes == expected_cpes + + +def test_related_to_hardware(): + assert ( + related_to_hardware( + cpes=[ + "cpe:2.3:a:csilvers:gperftools:0.1:*:*:*:*:*:*:*", + "cpe:2.3:h:csilvers:gperftools:0.2:*:*:*:*:*:*:*", + "cpe:2.3:a:csilvers:gperftools:*:*:*:*:*:*:*:*", + ] + ) + == True + ) + + +def test_extract_summary_with_single_summary(): + expected_summary = ( + "Multiple integer overflows in TCMalloc (tcmalloc.cc) in gperftools " + "before 0.4 make it easier for context-dependent attackers to perform memory-related " + "attacks such as buffer overflows via a large size value, which causes less memory to " + "be allocated than expected." + ) + found_summary = extract_summary(get_cve_item()) + assert found_summary == expected_summary + + +def test_extract_reference_urls(): + expected_urls = { + "http://code.google.com/p/gperftools/source/browse/tags/perftools-0.4/ChangeLog", + "http://kqueue.org/blog/2012/03/05/memory-allocator-security-revisited/", + } + + found_urls = extract_reference_urls(get_cve_item()) + + assert found_urls == expected_urls From c9c96c0ad6a6f6029fc6d561d9c142a2c97b3703 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 9 Apr 2022 10:06:31 +0200 Subject: [PATCH 4/5] Add link to NVD License issue Signed-off-by: Philippe Ombredanne --- vulnerabilities/importers/nvd.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vulnerabilities/importers/nvd.py b/vulnerabilities/importers/nvd.py index d66ef29ba..4f34dbb4e 100644 --- a/vulnerabilities/importers/nvd.py +++ b/vulnerabilities/importers/nvd.py @@ -41,6 +41,7 @@ class NVDImporter(Importer): + # See https://github.com/nexB/vulnerablecode/issues/665 for follow up spdx_license_expression = "LicenseRef-scancode-unknown" def advisory_data(self): From ec3e564f09ed5b80fcf7eb9c2ebe207635e5c7c3 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 9 Apr 2022 10:07:20 +0200 Subject: [PATCH 5/5] Set reference_id length to 200 Signed-off-by: Philippe Ombredanne --- .../0007_alter_vulnerabilityreference_reference_id.py | 2 +- vulnerabilities/models.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/migrations/0007_alter_vulnerabilityreference_reference_id.py b/vulnerabilities/migrations/0007_alter_vulnerabilityreference_reference_id.py index 14bb61f13..6060dc739 100644 --- a/vulnerabilities/migrations/0007_alter_vulnerabilityreference_reference_id.py +++ b/vulnerabilities/migrations/0007_alter_vulnerabilityreference_reference_id.py @@ -17,7 +17,7 @@ class Migration(migrations.Migration): field=models.CharField( blank=True, help_text="An optional reference ID, such as DSA-4465-1 when available", - max_length=180, + max_length=200, null=True, ), ), diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 3b039f7df..879c9cd5b 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -104,7 +104,7 @@ class VulnerabilityReference(models.Model): max_length=1024, help_text="URL to the vulnerability reference", blank=True ) reference_id = models.CharField( - max_length=180, + max_length=200, help_text="An optional reference ID, such as DSA-4465-1 when available", blank=True, null=True,