diff --git a/SOURCES.rst b/SOURCES.rst index 281f77182..1c8134e8c 100644 --- a/SOURCES.rst +++ b/SOURCES.rst @@ -47,3 +47,5 @@ +----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+ |suse_scores | https://ftp.suse.com/pub/projects/security/yaml/suse-cvss-scores.yaml |vulnerability severity scores by SUSE | +----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+ +|mozilla | https://github.com/mozilla/foundation-security-advisories |mozilla | ++----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+ diff --git a/pytest.ini b/pytest.ini index 7a196de40..dea1a69ea 100644 --- a/pytest.ini +++ b/pytest.ini @@ -30,5 +30,6 @@ addopts = --ignore=vulnerabilities/importers/suse_backports.py --ignore=vulnerabilities/importers/suse_scores.py --ignore=vulnerabilities/importers/ubuntu_usn.py + --ignore=vulnerabilities/importers/mozilla.py --ignore=vulnerabilities/management/commands/create_cpe_to_purl_map.py --ignore=vulnerabilities/lib_oval.py diff --git a/requirements.txt b/requirements.txt index 79f88a040..fb5cdb4cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,3 +17,5 @@ lxml>=4.6.4 gunicorn>=20.1.0 django-environ==0.4.5 defusedxml==0.7.1 + +Markdown==3.3.4 diff --git a/vulnerabilities/helpers.py b/vulnerabilities/helpers.py index 9022b6b97..b43cc0242 100644 --- a/vulnerabilities/helpers.py +++ b/vulnerabilities/helpers.py @@ -72,6 +72,41 @@ def fetch_yaml(url): create_etag = MagicMock() +def split_markdown_front_matter(lines: str) -> Tuple[str, str]: + """ + This function splits lines into markdown front matter and the markdown body + and returns list of lines for both + + for example : + lines = + --- + title: ISTIO-SECURITY-2019-001 + description: Incorrect access control. + cves: [CVE-2019-12243] + --- + # Markdown starts here + + split_markdown_front_matter(lines) would return + ['title: ISTIO-SECURITY-2019-001','description: Incorrect access control.' + ,'cves: [CVE-2019-12243]'], + ["# Markdown starts here"] + """ + + fmlines = [] + mdlines = [] + splitter = mdlines + + for index, line in enumerate(lines.split("\n")): + if index == 0 and line.strip().startswith("---"): + splitter = fmlines + elif line.strip().startswith("---"): + splitter = mdlines + else: + splitter.append(line) + + return "\n".join(fmlines), "\n".join(mdlines) + + def contains_alpha(string): """ Return True if the input 'string' contains any alphabet diff --git a/vulnerabilities/importers/mozilla.py b/vulnerabilities/importers/mozilla.py new file mode 100644 index 000000000..d3bbbd0c8 --- /dev/null +++ b/vulnerabilities/importers/mozilla.py @@ -0,0 +1,180 @@ +import re +from typing import List +from typing import Set + +import yaml +from bs4 import BeautifulSoup +from markdown import markdown +from packageurl import PackageURL + +from vulnerabilities.importer import Advisory +from vulnerabilities.importer import GitImporter +from vulnerabilities.importer import Reference +from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.helpers import is_cve +from vulnerabilities.helpers import split_markdown_front_matter +from vulnerabilities.severity_systems import SCORING_SYSTEMS + +REPOSITORY = "mozilla/foundation-security-advisories" +MFSA_FILENAME_RE = re.compile(r"mfsa(\d{4}-\d{2,3})\.(md|yml)$") + + +class MozillaImporter(GitImporter): + def __enter__(self): + super(MozillaImporter, self).__enter__() + + if not getattr(self, "_added_files", None): + self._added_files, self._updated_files = self.file_changes( + recursive=True, subdir="announce" + ) + + def updated_advisories(self) -> Set[Advisory]: + files = self._updated_files.union(self._added_files) + files = [ + f for f in files if f.endswith(".md") or f.endswith(".yml") + ] # skip irrelevant files + + advisories = [] + for path in files: + advisories.extend(to_advisories(path)) + + return self.batch_advisories(advisories) + + +def to_advisories(path: str) -> List[Advisory]: + """ + Convert a file to corresponding advisories. + This calls proper method to handle yml/md files. + """ + mfsa_id = mfsa_id_from_filename(path) + if not mfsa_id: + return [] + + with open(path) as lines: + if path.endswith(".md"): + return get_advisories_from_md(mfsa_id, lines) + if path.endswith(".yml"): + return get_advisories_from_yml(mfsa_id, lines) + + return [] + + +def get_advisories_from_yml(mfsa_id, lines) -> List[Advisory]: + advisories = [] + data = yaml.safe_load(lines) + data["mfsa_id"] = mfsa_id + + fixed_package_urls = get_package_urls(data.get("fixed_in")) + references = get_yml_references(data) + + if not data.get("advisories"): + return [] + + for cve, advisory in data["advisories"].items(): + # These may contain HTML tags + summary = BeautifulSoup(advisory.get("description", ""), features="lxml").get_text() + + advisories.append( + Advisory( + summary=summary, + vulnerability_id=cve if is_cve(cve) else "", + impacted_package_urls=[], + resolved_package_urls=fixed_package_urls, + references=references, + ) + ) + + return advisories + + +def get_advisories_from_md(mfsa_id, lines) -> List[Advisory]: + yamltext, mdtext = split_markdown_front_matter(lines.read()) + data = yaml.safe_load(yamltext) + data["mfsa_id"] = mfsa_id + + fixed_package_urls = get_package_urls(data.get("fixed_in")) + references = get_yml_references(data) + cves = re.findall(r"CVE-\d+-\d+", yamltext + mdtext, re.IGNORECASE) + for cve in cves: + references.append( + Reference( + reference_id=cve, + url=f"https://cve.mitre.org/cgi-bin/cvename.cgi?name={cve}", + ) + ) + + description = html_get_p_under_h3(markdown(mdtext), "description") + + return [ + Advisory( + summary=description, + vulnerability_id="", + impacted_package_urls=[], + resolved_package_urls=fixed_package_urls, + references=references, + ) + ] + + +def html_get_p_under_h3(html, h3: str): + soup = BeautifulSoup(html, features="lxml") + h3tag = soup.find("h3", text=lambda txt: txt.lower() == h3) + p = "" + if h3tag: + for tag in h3tag.next_siblings: + if tag.name: + if tag.name != "p": + break + p += tag.get_text() + return p + + +def mfsa_id_from_filename(filename): + match = MFSA_FILENAME_RE.search(filename) + if match: + return "mfsa" + match.group(1) + + return None + + +def get_package_urls(pkgs: List[str]) -> List[PackageURL]: + package_urls = [ + PackageURL( + type="mozilla", + # pkg is of the form "Firefox ESR 1.21" or "Thunderbird 2.21" + name=pkg.rsplit(None, 1)[0], + version=pkg.rsplit(None, 1)[1], + ) + for pkg in pkgs + if pkg + ] + return package_urls + + +def get_yml_references(data: any) -> List[Reference]: + """ + Returns a list of references + Currently only considers the given mfsa as a reference + """ + # FIXME: Needs improvement + # Should we add 'bugs' section in references too? + # Should we add 'impact'/severity of CVE in references too? + # If yes, then fix alpine_linux importer as well + # Otherwise, do we need severity field for adversary as well? + + severities = ["critical", "high", "medium", "low", "none"] + severity = "none" + if data.get("impact"): + impact = data.get("impact").lower() + for s in severities: + if s in impact: + severity = s + break + + return [ + Reference( + reference_id=data["mfsa_id"], + url="https://www.mozilla.org/en-US/security/advisories/{}".format(data["mfsa_id"]), + severities=[VulnerabilitySeverity(scoring_systems["generic_textual"], severity)], + ) + ] diff --git a/vulnerabilities/tests/conftest.py b/vulnerabilities/tests/conftest.py index 033b90539..d6072ff86 100644 --- a/vulnerabilities/tests/conftest.py +++ b/vulnerabilities/tests/conftest.py @@ -74,4 +74,5 @@ def no_rmtree(monkeypatch): "test_importer_yielder.py", "test_upstream.py", "test_istio.py", + "test_mozilla.py", ] diff --git a/vulnerabilities/tests/test_data/mozilla.zip b/vulnerabilities/tests/test_data/mozilla.zip new file mode 100644 index 000000000..a572c4e9a Binary files /dev/null and b/vulnerabilities/tests/test_data/mozilla.zip differ diff --git a/vulnerabilities/tests/test_mozilla.py b/vulnerabilities/tests/test_mozilla.py new file mode 100644 index 000000000..83e7435bc --- /dev/null +++ b/vulnerabilities/tests/test_mozilla.py @@ -0,0 +1,97 @@ +import os +import shutil +import tempfile +import zipfile +from unittest.mock import patch + +from django.test import TestCase + +from vulnerabilities import models +from vulnerabilities.import_runner import ImportRunner +from vulnerabilities.importers.npm import categorize_versions + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, "test_data/") + + +@patch("vulnerabilities.importers.MozillaImporter._update_from_remote") +class MozillaImportTest(TestCase): + + tempdir = None + + @classmethod + def setUpClass(cls) -> None: + cls.tempdir = tempfile.mkdtemp() + zip_path = os.path.join(TEST_DATA, "mozilla.zip") + + with zipfile.ZipFile(zip_path, "r") as zip_ref: + zip_ref.extractall(cls.tempdir) + + cls.importer = models.Importer.objects.create( + name="mozilla_unittests", + license="", + last_run=None, + data_source="MozillaImporter", + data_source_cfg={ + "repository_url": "https://example.git", + "working_directory": os.path.join(cls.tempdir, "mozilla_test"), + "create_working_directory": False, + "remove_working_directory": False, + }, + ) + + @classmethod + def tearDownClass(cls) -> None: + # Make sure no requests for unexpected package names have been made during the tests. + shutil.rmtree(cls.tempdir) + + def test_import(self, _): + runner = ImportRunner(self.importer, 100) + + # Remove if we don't need set_api in MozillaImporter + # with patch("vulnerabilities.importers.MozillaImporter.versions", new=MOCK_VERSION_API): + # with patch("vulnerabilities.importers.MozillaImporter.set_api"): + # runner.run() + runner.run() + + assert models.Vulnerability.objects.count() == 9 + assert models.VulnerabilityReference.objects.count() == 10 + assert models.VulnerabilitySeverity.objects.count() == 9 + assert models.PackageRelatedVulnerability.objects.filter(is_vulnerable=False).count() == 16 + + assert models.Package.objects.count() == 12 + + self.assert_for_package("Firefox ESR", "mfsa2021-06", "78.7.1") + self.assert_for_package("Firefox ESR", "mfsa2021-04", "78.7", "CVE-2021-23953") + self.assert_for_package("Firefox for Android", "mfsa2021-01", "84.1.3", "CVE-2020-16044") + self.assert_for_package("Thunderbird", "mfsa2014-30", "24.4") + self.assert_for_package("Thunderbird", "mfsa2014-30", "24.4") + self.assert_for_package("Mozilla Suite", "mfsa2005-29", "1.7.6") + + def assert_for_package( + self, + package_name, + mfsa_id, + resolved_version, + vulnerability_id=None, + impacted_version=None, + ): + vuln = None + + pkg = models.Package.objects.get(name=package_name, version=resolved_version) + vuln = pkg.vulnerabilities.first() + + if vulnerability_id: + assert vuln.vulnerability_id == vulnerability_id + + ref_url = f"https://www.mozilla.org/en-US/security/advisories/{mfsa_id}" + assert models.VulnerabilityReference.objects.get(url=ref_url, vulnerability=vuln) + + assert models.PackageRelatedVulnerability.objects.filter( + package=pkg, vulnerability=vuln, is_vulnerable=False + ) + + +def test_categorize_versions_ranges(): + # Populate if impacted version is filled + pass