diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index b9ccaa512..261106685 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -22,7 +22,13 @@ from vulnerabilities.importers import alpine_linux from vulnerabilities.importers import github from vulnerabilities.importers import nginx +from vulnerabilities.importers import nvd -IMPORTERS_REGISTRY = [nginx.NginxImporter, alpine_linux.AlpineImporter, github.GitHubAPIImporter] +IMPORTERS_REGISTRY = [ + nginx.NginxImporter, + alpine_linux.AlpineImporter, + github.GitHubAPIImporter, + nvd.NVDImporter, +] IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY} diff --git a/vulnerabilities/importers/nvd.py b/vulnerabilities/importers/nvd.py index c5e662416..4f34dbb4e 100644 --- a/vulnerabilities/importers/nvd.py +++ b/vulnerabilities/importers/nvd.py @@ -20,150 +20,189 @@ # VulnerableCode is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. -import dataclasses import gzip import json from datetime import date +from typing import Iterable import requests from dateutil import parser as dateparser +from django.db.models.query import QuerySet -from vulnerabilities.helpers import create_etag -from vulnerabilities.importer import Advisory +from vulnerabilities.helpers import get_item +from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.severity_systems import scoring_systems - -BASE_URL = "https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-{}.json.gz" +from vulnerabilities.improver import Improver +from vulnerabilities.improver import Inference +from vulnerabilities.models import Advisory +from vulnerabilities.severity_systems import SCORING_SYSTEMS class NVDImporter(Importer): - def updated_advisories(self): + # See https://github.com/nexB/vulnerablecode/issues/665 for follow up + spdx_license_expression = "LicenseRef-scancode-unknown" + + def advisory_data(self): + advisory_data = [] current_year = date.today().year # NVD json feeds start from 2002. for year in range(2002, current_year + 1): - download_url = BASE_URL.format(year) - # Etags are like hashes of web responses. We maintain - # (url, etag) mappings in the DB. `create_etag` creates - # (url, etag) pair. If a (url, etag) already exists then the code - # skips processing the response further to avoid duplicate work - if create_etag(data_src=self, url=download_url, etag_key="etag"): - data = self.fetch(download_url) - yield self.to_advisories(data) - - @staticmethod - def fetch(url): - gz_file = requests.get(url) - data = gzip.decompress(gz_file.content) - return json.loads(data) - - def to_advisories(self, nvd_data): - for cve_item in nvd_data["CVE_Items"]: - if self.is_outdated(cve_item): - continue - - if self.related_to_hardware(cve_item): - continue - - cve_id = cve_item["cve"]["CVE_data_meta"]["ID"] - ref_urls = self.extract_reference_urls(cve_item) - references = [Reference(url=url) for url in ref_urls] - severity_scores = self.extract_severity_scores(cve_item) + download_url = f"https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-{year}.json.gz" + data = fetch(download_url) + advisory_data.extend(to_advisories(data)) + return advisory_data + + +# Isolating network calls for simplicity of testing +def fetch(url): + gz_file = requests.get(url) + data = gzip.decompress(gz_file.content) + return json.loads(data) + + +def extract_summary(cve_item): + """ + Return a summary for a given CVE item. + """ + # In 99% of cases len(cve_item['cve']['description']['description_data']) == 1 , so + # this usually returns cve_item['cve']['description']['description_data'][0]['value'] + # In the remaining 1% cases this returns the longest summary. + summaries = [] + for desc in get_item(cve_item, "cve", "description", "description_data") or []: + if desc.get("value"): + summaries.append(desc["value"]) + return max(summaries, key=len) if summaries else None + + +def to_advisories(nvd_data): + """ + Yield AdvisoryData objects from a NVD json feed. + """ + for cve_item in nvd_data.get("CVE_Items") or []: + cpes = extract_cpes(cve_item) + if related_to_hardware(cpes): + continue + + aliases = [] + cve_id = get_item(cve_item, "cve", "CVE_data_meta", "ID") + ref_urls = extract_reference_urls(cve_item) + references = [] + severity_scores = list(extract_severity_scores(cve_item)) + for cpe in cpes: references.append( Reference( - url=f"https://nvd.nist.gov/vuln/detail/{cve_id}", - reference_id=cve_id, - severities=severity_scores, - ) - ) - summary = self.extract_summary(cve_item) - yield Advisory( - vulnerability_id=cve_id, - summary=summary, - references=references, - ) - - @staticmethod - def extract_summary(cve_item): - # In 99% of cases len(cve_item['cve']['description']['description_data']) == 1 , so - # this usually returns cve_item['cve']['description']['description_data'][0]['value'] - # In the remaining 1% cases this returns the longest summary. - summaries = [desc["value"] for desc in cve_item["cve"]["description"]["description_data"]] - return max(summaries, key=len) - - @staticmethod - def extract_severity_scores(cve_item): - severity_scores = [] - - if cve_item["impact"].get("baseMetricV3"): - severity_scores.append( - VulnerabilitySeverity( - system=scoring_systems["cvssv3"], - value=str(cve_item["impact"]["baseMetricV3"]["cvssV3"]["baseScore"]), + reference_id=cpe, ) ) - severity_scores.append( - VulnerabilitySeverity( - system=scoring_systems["cvssv3_vector"], - value=str(cve_item["impact"]["baseMetricV3"]["cvssV3"]["vectorString"]), - ) - ) - - if cve_item["impact"].get("baseMetricV2"): - severity_scores.append( - VulnerabilitySeverity( - system=scoring_systems["cvssv2"], - value=str(cve_item["impact"]["baseMetricV2"]["cvssV2"]["baseScore"]), - ) + references.append( + Reference( + url=f"https://nvd.nist.gov/vuln/detail/{cve_id}", + reference_id=cve_id, + severities=severity_scores, ) - severity_scores.append( - VulnerabilitySeverity( - system=scoring_systems["cvssv2_vector"], - value=str(cve_item["impact"]["baseMetricV2"]["cvssV2"]["vectorString"]), - ) + ) + if "https://nvd.nist.gov/vuln/detail/{cve_id}" in ref_urls: + ref_urls.remove(f"https://nvd.nist.gov/vuln/detail/{cve_id}") + references.extend([Reference(url=url) for url in ref_urls]) + if cve_id: + aliases.append(cve_id) + summary = extract_summary(cve_item) + yield AdvisoryData( + aliases=aliases, + summary=summary, + references=sorted(references), + date_published=dateparser.parse(cve_item.get("publishedDate")), + ) + + +def extract_reference_urls(cve_item): + """ + Return a list of reference URLs for a given CVE item. + """ + urls = set() + for reference in get_item(cve_item, "cve", "references", "reference_data") or []: + ref_url = reference.get("url") + + if not ref_url: + continue + + if ref_url.startswith( + ( + "http", + "ftp", ) - - return severity_scores - - def extract_reference_urls(self, cve_item): - urls = set() - for reference in cve_item["cve"]["references"]["reference_data"]: - ref_url = reference["url"] - - if not ref_url: - continue - - if ref_url.startswith("http") or ref_url.startswith("ftp"): - urls.add(ref_url) - - return urls - - def is_outdated(self, cve_item): - cve_last_modified_date = cve_item["lastModifiedDate"] - cve_last_modified_date_obj = dateparser.parse(cve_last_modified_date) - - if self.config.cutoff_date: - return cve_last_modified_date_obj < self.config.cutoff_date - - if self.config.last_run_date: - return cve_last_modified_date_obj < self.config.last_run_date - - return False - - def related_to_hardware(self, cve_item): - for cpe in self.extract_cpes(cve_item): - cpe_comps = cpe.split(":") - # CPE follow the format cpe:cpe_version:product_type:vendor:product - if cpe_comps[2] == "h": - return True - - return False - - @staticmethod - def extract_cpes(cve_item): - cpes = set() - for node in cve_item["configurations"]["nodes"]: - for cpe_data in node.get("cpe_match", []): - cpes.add(cpe_data["cpe23Uri"]) - return cpes + ): + urls.add(ref_url) + + return urls + + +def related_to_hardware(cpes): + """ + Return True if the CVE item is related to hardware. + """ + for cpe in cpes: + cpe_comps = cpe.split(":") + # CPE follow the format cpe:cpe_version:product_type:vendor:product + if len(cpe_comps) > 2 and cpe_comps[2] == "h": + return True + + return False + + +def extract_cpes(cve_item): + """ + Return a list of CPEs for a given CVE item. + """ + cpes = set() + for node in get_item(cve_item, "configurations", "nodes") or []: + for cpe_data in node.get("cpe_match") or []: + cpe23_uri = cpe_data.get("cpe23Uri") + if cpe23_uri: + cpes.add(cpe23_uri) + return cpes + + +def extract_severity_scores(cve_item): + """ + Yield a vulnerability severity for each `cve_item`. + """ + if not isinstance(cve_item, dict): + return None + impact = cve_item.get("impact") or {} + base_metric_v3 = impact.get("baseMetricV3") or {} + if base_metric_v3: + cvss_v3 = get_item(base_metric_v3, "cvssV3") + yield VulnerabilitySeverity( + system=SCORING_SYSTEMS["cvssv3"], + value=str(cvss_v3.get("baseScore") or ""), + ) + yield VulnerabilitySeverity( + system=SCORING_SYSTEMS["cvssv3_vector"], + value=str(cvss_v3.get("vectorString") or ""), + ) + + base_metric_v2 = impact.get("baseMetricV2") or {} + if base_metric_v2: + cvss_v2 = base_metric_v2.get("cvssV2") or {} + yield VulnerabilitySeverity( + system=SCORING_SYSTEMS["cvssv2"], + value=str(cvss_v2.get("baseScore") or ""), + ) + yield VulnerabilitySeverity( + system=SCORING_SYSTEMS["cvssv2_vector"], + value=str(cvss_v2.get("vectorString") or ""), + ) + + +class NVDBasicImprover(Improver): + @property + def interesting_advisories(self) -> QuerySet: + return Advisory.objects.filter(created_by=NVDImporter.qualified_name) + + def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]: + yield Inference.from_advisory_data( + advisory_data=advisory_data, confidence=100, fixed_purl=None + ) diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index ce640c840..0dbb2a424 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -6,6 +6,7 @@ importers.nginx.NginxBasicImprover, importers.alpine_linux.AlpineBasicImprover, importers.github.GitHubBasicImprover, + importers.nvd.NVDBasicImprover, ] IMPROVERS_REGISTRY = {x.qualified_name: x for x in IMPROVERS_REGISTRY} diff --git a/vulnerabilities/migrations/0007_alter_vulnerabilityreference_reference_id.py b/vulnerabilities/migrations/0007_alter_vulnerabilityreference_reference_id.py new file mode 100644 index 000000000..6060dc739 --- /dev/null +++ b/vulnerabilities/migrations/0007_alter_vulnerabilityreference_reference_id.py @@ -0,0 +1,24 @@ +# Generated by Django 4.0.2 on 2022-04-08 18:53 + +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0006_alter_advisory_unique_together"), + ] + + operations = [ + migrations.AlterField( + model_name="vulnerabilityreference", + name="reference_id", + field=models.CharField( + blank=True, + help_text="An optional reference ID, such as DSA-4465-1 when available", + max_length=200, + null=True, + ), + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index e8062fa7c..879c9cd5b 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -104,7 +104,7 @@ class VulnerabilityReference(models.Model): max_length=1024, help_text="URL to the vulnerability reference", blank=True ) reference_id = models.CharField( - max_length=50, + max_length=200, help_text="An optional reference ID, such as DSA-4465-1 when available", blank=True, null=True, diff --git a/vulnerabilities/tests/conftest.py b/vulnerabilities/tests/conftest.py index 58ad04b38..0bb246db3 100644 --- a/vulnerabilities/tests/conftest.py +++ b/vulnerabilities/tests/conftest.py @@ -47,7 +47,6 @@ def no_rmtree(monkeypatch): "test_apache_httpd.py", "test_npm.py", "test_apache_kafka.py", - "test_nvd.py", "test_apache_tomcat.py", "test_openssl.py", "test_api.py", diff --git a/vulnerabilities/tests/test_cpe_reference.py b/vulnerabilities/tests/test_cpe_reference.py new file mode 100644 index 000000000..3c9440417 --- /dev/null +++ b/vulnerabilities/tests/test_cpe_reference.py @@ -0,0 +1,37 @@ +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. +import pytest + +from vulnerabilities.improve_runner import get_or_create_vulnerability_and_aliases +from vulnerabilities.models import Vulnerability +from vulnerabilities.models import VulnerabilityReference + + +@pytest.mark.django_db +def test_cpe_as_reference_id_in_db(): + vulnerability = Vulnerability(summary="lorem ipsum" * 10) + vulnerability.save() + VulnerabilityReference.objects.get_or_create( + reference_id="cpe:2.3:a:microsoft:windows_10:10.0.17134:*:*:*:*:*:*:*" * 3, + url="https://foo.com", + vulnerability=vulnerability, + ) diff --git a/vulnerabilities/tests/test_data/nvd/nvd-expected.json b/vulnerabilities/tests/test_data/nvd/nvd-expected.json new file mode 100644 index 000000000..39bb178ae --- /dev/null +++ b/vulnerabilities/tests/test_data/nvd/nvd-expected.json @@ -0,0 +1,335 @@ +[ + { + "aliases": [ + "CVE-2005-4895" + ], + "summary": "Multiple integer overflows in TCMalloc (tcmalloc.cc) in gperftools before 0.4 make it easier for context-dependent attackers to perform memory-related attacks such as buffer overflows via a large size value, which causes less memory to be allocated than expected.", + "affected_packages": [], + "references": [ + { + "reference_id": "", + "url": "http://code.google.com/p/gperftools/source/browse/tags/perftools-0.4/ChangeLog", + "severities": [] + }, + { + "reference_id": "", + "url": "http://kqueue.org/blog/2012/03/05/memory-allocator-security-revisited/", + "severities": [] + }, + { + "reference_id": "CVE-2005-4895", + "url": "https://nvd.nist.gov/vuln/detail/CVE-2005-4895", + "severities": [ + { + "system": "cvssv2", + "value": "5.0" + }, + { + "system": "cvssv2_vector", + "value": "AV:N/AC:L/Au:N/C:N/I:N/A:P" + } + ] + }, + { + "reference_id": "cpe:2.3:a:csilvers:gperftools:*:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:a:csilvers:gperftools:0.1:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:a:csilvers:gperftools:0.2:*:*:*:*:*:*:*", + "url": "", + "severities": [] + } + ], + "date_published": "2012-07-25T19:55:00+00:00" + }, + { + "aliases": [ + "CVE-2003-0001" + ], + "summary": "Multiple ethernet Network Interface Card (NIC) device drivers do not pad frames with null bytes, which allows remote attackers to obtain information from previous packets or kernel memory by using malformed packets, as demonstrated by Etherleak.", + "affected_packages": [], + "references": [ + { + "reference_id": "", + "url": "http://archives.neohapsis.com/archives/vulnwatch/2003-q1/0016.html", + "severities": [] + }, + { + "reference_id": "", + "url": "http://marc.info/?l=bugtraq&m=104222046632243&w=2", + "severities": [] + }, + { + "reference_id": "", + "url": "http://secunia.com/advisories/7996", + "severities": [] + }, + { + "reference_id": "", + "url": "http://www.atstake.com/research/advisories/2003/a010603-1.txt", + "severities": [] + }, + { + "reference_id": "", + "url": "http://www.atstake.com/research/advisories/2003/atstake_etherleak_report.pdf", + "severities": [] + }, + { + "reference_id": "", + "url": "http://www.kb.cert.org/vuls/id/412115", + "severities": [] + }, + { + "reference_id": "", + "url": "http://www.oracle.com/technetwork/topics/security/cpujan2015-1972971.html", + "severities": [] + }, + { + "reference_id": "", + "url": "http://www.osvdb.org/9962", + "severities": [] + }, + { + "reference_id": "", + "url": "http://www.redhat.com/support/errata/RHSA-2003-025.html", + "severities": [] + }, + { + "reference_id": "", + "url": "http://www.redhat.com/support/errata/RHSA-2003-088.html", + "severities": [] + }, + { + "reference_id": "", + "url": "http://www.securityfocus.com/archive/1/305335/30/26420/threaded", + "severities": [] + }, + { + "reference_id": "", + "url": "http://www.securityfocus.com/archive/1/307564/30/26270/threaded", + "severities": [] + }, + { + "reference_id": "", + "url": "http://www.securitytracker.com/id/1031583", + "severities": [] + }, + { + "reference_id": "", + "url": "http://www.securitytracker.com/id/1040185", + "severities": [] + }, + { + "reference_id": "", + "url": "https://oval.cisecurity.org/repository/search/definition/oval%3Aorg.mitre.oval%3Adef%3A2665", + "severities": [] + }, + { + "reference_id": "CVE-2003-0001", + "url": "https://nvd.nist.gov/vuln/detail/CVE-2003-0001", + "severities": [ + { + "system": "cvssv2", + "value": "5.0" + }, + { + "system": "cvssv2_vector", + "value": "AV:N/AC:L/Au:N/C:P/I:N/A:N" + } + ] + }, + { + "reference_id": "cpe:2.3:o:freebsd:freebsd:4.2:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:freebsd:freebsd:4.3:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:freebsd:freebsd:4.4:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:freebsd:freebsd:4.5:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:freebsd:freebsd:4.6:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:freebsd:freebsd:4.7:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.10:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.11:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.12:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.13:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.14:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.15:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.16:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.17:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.18:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.19:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.1:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.20:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.2:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.3:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.4:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.5:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.6:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.7:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.8:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:linux:linux_kernel:2.4.9:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:microsoft:windows_2000:*:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:microsoft:windows_2000:*:sp1:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:microsoft:windows_2000:*:sp2:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:microsoft:windows_2000_terminal_services:*:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:microsoft:windows_2000_terminal_services:*:sp1:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:microsoft:windows_2000_terminal_services:*:sp2:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:netbsd:netbsd:1.5.1:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:netbsd:netbsd:1.5.2:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:netbsd:netbsd:1.5.3:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:netbsd:netbsd:1.5:*:*:*:*:*:*:*", + "url": "", + "severities": [] + }, + { + "reference_id": "cpe:2.3:o:netbsd:netbsd:1.6:*:*:*:*:*:*:*", + "url": "", + "severities": [] + } + ], + "date_published": "2003-01-17T05:00:00+00:00" + } +] \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/nvd/nvd_test.json b/vulnerabilities/tests/test_data/nvd/nvd_test.json index a91a8fbf4..5bb675aa5 100644 --- a/vulnerabilities/tests/test_data/nvd/nvd_test.json +++ b/vulnerabilities/tests/test_data/nvd/nvd_test.json @@ -264,6 +264,289 @@ }, "publishedDate": "2016-10-14T16:59Z", "lastModifiedDate": "2018-05-30T01:29Z" - } + }, + { + "cve" : { + "data_type" : "CVE", + "data_format" : "MITRE", + "data_version" : "4.0", + "CVE_data_meta" : { + "ID" : "CVE-2003-0001", + "ASSIGNER" : "cve@mitre.org" + }, + "problemtype" : { + "problemtype_data" : [ { + "description" : [ { + "lang" : "en", + "value" : "CWE-200" + } ] + } ] + }, + "references" : { + "reference_data" : [ { + "url" : "http://www.atstake.com/research/advisories/2003/a010603-1.txt", + "name" : "A010603-1", + "refsource" : "ATSTAKE", + "tags" : [ "Vendor Advisory" ] + }, { + "url" : "http://www.kb.cert.org/vuls/id/412115", + "name" : "VU#412115", + "refsource" : "CERT-VN", + "tags" : [ "Third Party Advisory", "US Government Resource" ] + }, { + "url" : "http://archives.neohapsis.com/archives/vulnwatch/2003-q1/0016.html", + "name" : "20030110 More information regarding Etherleak", + "refsource" : "VULNWATCH", + "tags" : [ ] + }, { + "url" : "http://www.atstake.com/research/advisories/2003/atstake_etherleak_report.pdf", + "name" : "http://www.atstake.com/research/advisories/2003/atstake_etherleak_report.pdf", + "refsource" : "MISC", + "tags" : [ ] + }, { + "url" : "http://www.redhat.com/support/errata/RHSA-2003-025.html", + "name" : "RHSA-2003:025", + "refsource" : "REDHAT", + "tags" : [ ] + }, { + "url" : "http://www.redhat.com/support/errata/RHSA-2003-088.html", + "name" : "RHSA-2003:088", + "refsource" : "REDHAT", + "tags" : [ ] + }, { + "url" : "http://www.osvdb.org/9962", + "name" : "9962", + "refsource" : "OSVDB", + "tags" : [ ] + }, { + "url" : "http://secunia.com/advisories/7996", + "name" : "7996", + "refsource" : "SECUNIA", + "tags" : [ ] + }, { + "url" : "http://www.oracle.com/technetwork/topics/security/cpujan2015-1972971.html", + "name" : "http://www.oracle.com/technetwork/topics/security/cpujan2015-1972971.html", + "refsource" : "CONFIRM", + "tags" : [ ] + }, { + "url" : "http://marc.info/?l=bugtraq&m=104222046632243&w=2", + "name" : "20030110 More information regarding Etherleak", + "refsource" : "BUGTRAQ", + "tags" : [ ] + }, { + "url" : "http://www.securitytracker.com/id/1031583", + "name" : "1031583", + "refsource" : "SECTRACK", + "tags" : [ ] + }, { + "url" : "https://oval.cisecurity.org/repository/search/definition/oval%3Aorg.mitre.oval%3Adef%3A2665", + "name" : "oval:org.mitre.oval:def:2665", + "refsource" : "OVAL", + "tags" : [ ] + }, { + "url" : "http://www.securitytracker.com/id/1040185", + "name" : "1040185", + "refsource" : "SECTRACK", + "tags" : [ ] + }, { + "url" : "http://www.securityfocus.com/archive/1/307564/30/26270/threaded", + "name" : "20030117 Re: More information regarding Etherleak", + "refsource" : "BUGTRAQ", + "tags" : [ ] + }, { + "url" : "http://www.securityfocus.com/archive/1/305335/30/26420/threaded", + "name" : "20030106 Etherleak: Ethernet frame padding information leakage (A010603-1)", + "refsource" : "BUGTRAQ", + "tags" : [ ] + } ] + }, + "description" : { + "description_data" : [ { + "lang" : "en", + "value" : "Multiple ethernet Network Interface Card (NIC) device drivers do not pad frames with null bytes, which allows remote attackers to obtain information from previous packets or kernel memory by using malformed packets, as demonstrated by Etherleak." + } ] + } + }, + "configurations" : { + "CVE_data_version" : "4.0", + "nodes" : [ { + "operator" : "OR", + "children" : [ ], + "cpe_match" : [ { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.1:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:freebsd:freebsd:4.6:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:freebsd:freebsd:4.7:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.15:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.16:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.4:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.5:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.6:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:microsoft:windows_2000:*:sp1:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:microsoft:windows_2000_terminal_services:*:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:netbsd:netbsd:1.6:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:freebsd:freebsd:4.2:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:freebsd:freebsd:4.3:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.11:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.12:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.19:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.2:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.9:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:microsoft:windows_2000:*:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:microsoft:windows_2000:*:sp2:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:netbsd:netbsd:1.5:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:netbsd:netbsd:1.5.1:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.10:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.17:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.18:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.7:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.8:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:microsoft:windows_2000_terminal_services:*:sp1:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:microsoft:windows_2000_terminal_services:*:sp2:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:freebsd:freebsd:4.4:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:freebsd:freebsd:4.5:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.13:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.14:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.20:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:linux:linux_kernel:2.4.3:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:netbsd:netbsd:1.5.2:*:*:*:*:*:*:*", + "cpe_name" : [ ] + }, { + "vulnerable" : true, + "cpe23Uri" : "cpe:2.3:o:netbsd:netbsd:1.5.3:*:*:*:*:*:*:*", + "cpe_name" : [ ] + } ] + } ] + }, + "impact" : { + "baseMetricV2" : { + "cvssV2" : { + "version" : "2.0", + "vectorString" : "AV:N/AC:L/Au:N/C:P/I:N/A:N", + "accessVector" : "NETWORK", + "accessComplexity" : "LOW", + "authentication" : "NONE", + "confidentialityImpact" : "PARTIAL", + "integrityImpact" : "NONE", + "availabilityImpact" : "NONE", + "baseScore" : 5.0 + }, + "severity" : "MEDIUM", + "exploitabilityScore" : 10.0, + "impactScore" : 2.9, + "obtainAllPrivilege" : false, + "obtainUserPrivilege" : false, + "obtainOtherPrivilege" : false, + "userInteractionRequired" : false + } + }, + "publishedDate" : "2003-01-17T05:00Z", + "lastModifiedDate" : "2019-04-30T14:27Z" + } ] } \ No newline at end of file diff --git a/vulnerabilities/tests/test_nvd.py b/vulnerabilities/tests/test_nvd.py index 2cd7856d7..960dfaef9 100644 --- a/vulnerabilities/tests/test_nvd.py +++ b/vulnerabilities/tests/test_nvd.py @@ -22,148 +22,167 @@ import json import os -from unittest import TestCase -from dateutil import parser as dateparser - -from vulnerabilities.importer import Advisory -from vulnerabilities.importer import Reference -from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.importers import NVDImporter -from vulnerabilities.severity_systems import scoring_systems +from vulnerabilities.importers.nvd import extract_cpes +from vulnerabilities.importers.nvd import extract_reference_urls +from vulnerabilities.importers.nvd import extract_summary +from vulnerabilities.importers.nvd import related_to_hardware +from vulnerabilities.importers.nvd import to_advisories BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data/nvd/nvd_test.json") -class TestNVDImporter(TestCase): - @classmethod - def setUpClass(cls): - data_source_cfg = {"etags": {}} - cls.data_src = NVDImporter(1, config=data_source_cfg) - with open(TEST_DATA) as f: - cls.nvd_data = json.load(f) - - def test_extract_cpes(self): - expected_cpes = { - "cpe:2.3:a:csilvers:gperftools:0.1:*:*:*:*:*:*:*", - "cpe:2.3:a:csilvers:gperftools:0.2:*:*:*:*:*:*:*", - "cpe:2.3:h:google:chrome:*:*:*:*:*:*:*:*", - "cpe:2.3:a:csilvers:gperftools:*:*:*:*:*:*:*:*", - } - - found_cpes = set() - for cve_item in self.nvd_data["CVE_Items"]: - found_cpes.update(NVDImporter.extract_cpes(cve_item)) - - assert expected_cpes == found_cpes - - def test_related_to_hardware(self): - # Only CVE-2005-4900 is supposed to be a hardware related - # vulnerability. - for cve_item in self.nvd_data["CVE_Items"]: - expected_result = "CVE-2005-4900" == cve_item["cve"]["CVE_data_meta"]["ID"] - assert self.data_src.related_to_hardware(cve_item) == expected_result - - def test_extract_summary_with_single_summary(self): - expected_summary = ( - "Multiple integer overflows in TCMalloc (tcmalloc.cc) in gperftools " - "before 0.4 make it easier for context-dependent attackers to perform memory-related " - "attacks such as buffer overflows via a large size value, which causes less memory to " - "be allocated than expected." - ) - cve_item = self.nvd_data["CVE_Items"][0] - assert len(cve_item["cve"]["description"]["description_data"]) == 1 - found_summary = NVDImporter.extract_summary(cve_item) - assert found_summary == expected_summary - - def test_extract_summary_with_multiple_summary(self): - expected_summary = ( - "SHA-1 is not collision resistant, which makes it easier for context-dependent " - "attackers to conduct spoofing attacks, as demonstrated by attacks on the use of SHA-1" - " in TLS 1.2. NOTE: this CVE exists to provide a common identifier for referencing " - "this SHA-1 issue; the existence of an identifier is not, by itself, a technology " - "recommendation." +def load_test_data(): + with open(TEST_DATA) as f: + return json.load(f) + + +def test_nvd_importer_with_hardware(regen=False): + expected_file = os.path.join(BASE_DIR, "test_data/nvd/nvd-expected.json") + + result = [data.to_dict() for data in list(to_advisories(load_test_data()))] + + if regen: + with open(expected_file, "w") as f: + json.dump(result, f, indent=2) + expected = result + else: + with open(expected_file) as f: + expected = json.load(f) + + assert result == expected + + +def get_cve_item(): + + return { + "cve": { + "data_type": "CVE", + "data_format": "MITRE", + "data_version": "4.0", + "CVE_data_meta": {"ID": "CVE-2005-4895", "ASSIGNER": "cve@mitre.org"}, + "problemtype": { + "problemtype_data": [{"description": [{"lang": "en", "value": "CWE-189"}]}] + }, + "references": { + "reference_data": [ + { + "url": "http://code.google.com/p/gperftools/source/browse/tags/perftools-0.4/ChangeLog", + "name": "http://code.google.com/p/gperftools/source/browse/tags/perftools-0.4/ChangeLog", + "refsource": "CONFIRM", + "tags": [], + }, + { + "url": "http://kqueue.org/blog/2012/03/05/memory-allocator-security-revisited/", + "name": "http://kqueue.org/blog/2012/03/05/memory-allocator-security-revisited/", + "refsource": "MISC", + "tags": [], + }, + ] + }, + "description": { + "description_data": [ + { + "lang": "en", + "value": "Multiple integer overflows in TCMalloc (tcmalloc.cc) in gperftools before 0.4 make it easier for context-dependent attackers to perform memory-related attacks such as buffer overflows via a large size value, which causes less memory to be allocated than expected.", + } + ] + }, + }, + "configurations": { + "CVE_data_version": "4.0", + "nodes": [ + { + "operator": "OR", + "cpe_match": [ + { + "vulnerable": True, + "cpe23Uri": "cpe:2.3:a:csilvers:gperftools:0.1:*:*:*:*:*:*:*", + }, + { + "vulnerable": True, + "cpe23Uri": "cpe:2.3:a:csilvers:gperftools:0.2:*:*:*:*:*:*:*", + }, + { + "vulnerable": True, + "cpe23Uri": "cpe:2.3:a:csilvers:gperftools:*:*:*:*:*:*:*:*", + "versionEndIncluding": "0.3", + }, + ], + } + ], + }, + "impact": { + "baseMetricV2": { + "cvssV2": { + "version": "2.0", + "vectorString": "AV:N/AC:L/Au:N/C:N/I:N/A:P", + "accessVector": "NETWORK", + "accessComplexity": "LOW", + "authentication": "NONE", + "confidentialityImpact": "NONE", + "integrityImpact": "NONE", + "availabilityImpact": "PARTIAL", + "baseScore": 5.0, + }, + "severity": "MEDIUM", + "exploitabilityScore": 10.0, + "impactScore": 2.9, + "obtainAllPrivilege": False, + "obtainUserPrivilege": False, + "obtainOtherPrivilege": False, + "userInteractionRequired": False, + } + }, + "publishedDate": "2012-07-25T19:55Z", + "lastModifiedDate": "2012-08-09T04:00Z", + } + + +def test_extract_cpes(): + expected_cpes = { + "cpe:2.3:a:csilvers:gperftools:0.1:*:*:*:*:*:*:*", + "cpe:2.3:a:csilvers:gperftools:0.2:*:*:*:*:*:*:*", + "cpe:2.3:a:csilvers:gperftools:*:*:*:*:*:*:*:*", + } + + found_cpes = set() + found_cpes.update(extract_cpes(get_cve_item())) + + assert found_cpes == expected_cpes + + +def test_related_to_hardware(): + assert ( + related_to_hardware( + cpes=[ + "cpe:2.3:a:csilvers:gperftools:0.1:*:*:*:*:*:*:*", + "cpe:2.3:h:csilvers:gperftools:0.2:*:*:*:*:*:*:*", + "cpe:2.3:a:csilvers:gperftools:*:*:*:*:*:*:*:*", + ] ) - cve_item = self.nvd_data["CVE_Items"][1] - assert len(cve_item["cve"]["description"]["description_data"]) > 1 - found_summary = NVDImporter.extract_summary(cve_item) - assert found_summary == expected_summary - - def test_is_outdated(self): - cve_item = self.nvd_data["CVE_Items"][0] - assert self.data_src.is_outdated(cve_item) is False - - self.data_src.config.cutoff_date = dateparser.parse("2019-08-05 13:14:17.733232+05:30") - assert self.data_src.is_outdated(cve_item) - self.data_src.config.cutoff_date = None # cleanup - - assert self.data_src.is_outdated(cve_item) is False - - self.data_src.config.last_run_date = dateparser.parse("2019-08-05 13:14:17.733232+05:30") - assert self.data_src.is_outdated(cve_item) - - self.data_src.config.last_run_date = dateparser.parse("2000-08-05 13:14:17.733232+05:30") - assert self.data_src.is_outdated(cve_item) is False - self.data_src.config.last_run_date = None # cleanup - - def test_extract_reference_urls(self): - cve_item = self.nvd_data["CVE_Items"][1] - expected_urls = { - "http://ia.cr/2007/474", - "http://shattered.io/", - "http://www.cwi.nl/news/2017/cwi-and-google-announce-first-collision-industry-security-standard-sha-1", # nopep8 - "http://www.securityfocus.com/bid/12577", - "https://arstechnica.com/security/2017/02/at-deaths-door-for-years-widely-used-sha1-function-is-now-dead/", # nopep8 - "https://security.googleblog.com/2015/12/an-update-on-sha-1-certificates-in.html", - "https://security.googleblog.com/2017/02/announcing-first-sha1-collision.html", - "https://sites.google.com/site/itstheshappening", - "https://www.schneier.com/blog/archives/2005/02/sha1_broken.html", - "https://www.schneier.com/blog/archives/2005/08/new_cryptanalyt.html", - } - - found_urls = self.data_src.extract_reference_urls(cve_item) - - assert found_urls == expected_urls - - def test_to_advisories(self): - - expected_advisories = [ - Advisory( - summary=( - "Multiple integer overflows in TCMalloc (tcmalloc.cc) in gperftools " - "before 0.4 make it easier for context-dependent attackers to perform memory-related " # nopep8 - "attacks such as buffer overflows via a large size value, which causes less memory to " # nopep8 - "be allocated than expected." - ), - references=[ - Reference( - url="http://code.google.com/p/gperftools/source/browse/tags/perftools-0.4/ChangeLog", # nopep8 - ), - Reference( - url="http://kqueue.org/blog/2012/03/05/memory-allocator-security-revisited/", # nopep8 - ), - Reference( - url="https://nvd.nist.gov/vuln/detail/CVE-2005-4895", - severities=[ - VulnerabilitySeverity( - system=scoring_systems["cvssv2"], - value="5.0", - ), - VulnerabilitySeverity( - system=scoring_systems["cvssv2_vector"], - value="AV:N/AC:L/Au:N/C:N/I:N/A:P", - ), - ], - reference_id="CVE-2005-4895", - ), - ], - vulnerability_id="CVE-2005-4895", - ) - ] - assert len(self.nvd_data["CVE_Items"]) == 2 - - found_advisories = list(self.data_src.to_advisories(self.nvd_data)) - found_advisories = list(map(Advisory.normalized, found_advisories)) - expected_advisories = list(map(Advisory.normalized, expected_advisories)) - assert sorted(found_advisories) == sorted(expected_advisories) + == True + ) + + +def test_extract_summary_with_single_summary(): + expected_summary = ( + "Multiple integer overflows in TCMalloc (tcmalloc.cc) in gperftools " + "before 0.4 make it easier for context-dependent attackers to perform memory-related " + "attacks such as buffer overflows via a large size value, which causes less memory to " + "be allocated than expected." + ) + found_summary = extract_summary(get_cve_item()) + assert found_summary == expected_summary + + +def test_extract_reference_urls(): + expected_urls = { + "http://code.google.com/p/gperftools/source/browse/tags/perftools-0.4/ChangeLog", + "http://kqueue.org/blog/2012/03/05/memory-allocator-security-revisited/", + } + + found_urls = extract_reference_urls(get_cve_item()) + + assert found_urls == expected_urls