From 4fbd1a5b10bc04ad4cd9c362986ba330b66309c1 Mon Sep 17 00:00:00 2001 From: ziad Date: Mon, 1 Aug 2022 15:48:48 +0200 Subject: [PATCH 1/8] Refactor Gitimporter using fetchcode Signed-off-by: ziad --- vulnerabilities/importer.py | 200 +++++----------------------- vulnerabilities/importers/gitlab.py | 32 +++-- 2 files changed, 51 insertions(+), 181 deletions(-) diff --git a/vulnerabilities/importer.py b/vulnerabilities/importer.py index 6fdfb3ef2..4826a6fb1 100644 --- a/vulnerabilities/importer.py +++ b/vulnerabilities/importer.py @@ -24,6 +24,7 @@ from typing import Tuple from binaryornot.helpers import is_binary_string +from fetchcode.vcs import fetch_via_vcs from git import DiffIndex from git import Repo from license_expression import Licensing @@ -312,193 +313,64 @@ def advisory_data(self) -> Iterable[AdvisoryData]: raise NotImplementedError -# TODO: Needs rewrite -class GitImporter(Importer): - def validate_configuration(self) -> None: +@dataclasses.dataclass +class GitConfig: + repo_url: str + save_working_directory: bool = True - if not self.config.create_working_directory and self.config.working_directory is None: - self.error( - '"create_working_directory" is not set but "working_directory" is set to ' - "the default, which calls tempfile.mkdtemp()" - ) - if not self.config.create_working_directory and not os.path.exists( - self.config.working_directory - ): - self.error( - '"working_directory" does not contain an existing directory and' - '"create_working_directory" is not set' - ) +class ForkError(Exception): + pass - if not self.config.remove_working_directory and self.config.working_directory is None: - self.error( - '"remove_working_directory" is not set and "working_directory" is set to ' - "the default, which calls tempfile.mkdtemp()" - ) - def __enter__(self): - self._ensure_working_directory() - self._ensure_repository() +class GitImporter(Importer): + def __init__(self, config): + super().__init__() + self.config = config + try: + self.vcs_response = fetch_via_vcs(self.config.repo_url) + except Exception as e: + logger.error(f"Can't clone url {self.config.repo_url} - {e}") + raise ForkError - def __exit__(self, exc_type, exc_val, exc_tb): - if self.config.remove_working_directory: - shutil.rmtree(self.config.working_directory) + def __exit__(self): + if not self.config.save_working_directory: + shutil.rmtree(self.vcs_response.dest_dir) - def file_changes( + def collect_files( self, subdir: str = None, recursive: bool = False, file_ext: Optional[str] = None, - ) -> Tuple[Set[str], Set[str]]: + ) -> Set[Path]: """ - Returns all added and modified files since last_run_date or cutoff_date (whichever is more - recent). - + Returns Set of all file names :param subdir: filter by files in this directory :param recursive: whether to include files in subdirectories :param file_ext: filter files by this extension - :return: The first set contains (absolute paths to) added files, the second one modified - files """ if subdir is None: - working_dir = self.config.working_directory + working_dir = self.vcs_response.dest_dir else: - working_dir = os.path.join(self.config.working_directory, subdir) + working_dir = os.path.join(self.vcs_response.dest_dir, subdir) path = Path(working_dir) - if self.config.last_run_date is None and self.config.cutoff_date is None: - if recursive: - glob = "**/*" - else: - glob = "*" - - if file_ext: - glob = f"{glob}.{file_ext}" - - return {str(p) for p in path.glob(glob) if p.is_file()}, set() - - return self._collect_file_changes(subdir=subdir, recursive=recursive, file_ext=file_ext) - - def _collect_file_changes( - self, - subdir: Optional[str], - recursive: bool, - file_ext: Optional[str], - ) -> Tuple[Set[str], Set[str]]: - - added_files, updated_files = set(), set() - - # find the most ancient commit we need to diff with - cutoff_commit = None - for commit in self._repo.iter_commits(self._repo.head): - if commit.committed_date < self.cutoff_timestamp: - break - cutoff_commit = commit - - if cutoff_commit is None: - return added_files, updated_files - - def _is_binary(d: DiffIndex): - return is_binary_string(d.b_blob.data_stream.read(1024)) - - for d in cutoff_commit.diff(self._repo.head.commit): - if not _include_file(d.b_path, subdir, recursive, file_ext) or _is_binary(d): - continue - - abspath = os.path.join(self.config.working_directory, d.b_path) - if d.new_file: - added_files.add(abspath) - elif d.a_blob and d.b_blob: - if d.a_path != d.b_path: - # consider moved files as added - added_files.add(abspath) - elif d.a_blob != d.b_blob: - updated_files.add(abspath) - - # Any file that has been added and then updated inside the window of the git history we - # looked at, should be considered "added", not "updated", since it does not exist in the - # database yet. - updated_files = updated_files - added_files - - return added_files, updated_files - - def _ensure_working_directory(self) -> None: - if self.config.working_directory is None: - self.config.working_directory = tempfile.mkdtemp() - elif self.config.create_working_directory and not os.path.exists( - self.config.working_directory - ): - os.mkdir(self.config.working_directory) - - def _ensure_repository(self) -> None: - if not os.path.exists(os.path.join(self.config.working_directory, ".git")): - self._clone_repository() - return - self._repo = Repo(self.config.working_directory) - - if self.config.branch is None: - self.config.branch = str(self._repo.active_branch) - branch = self.config.branch - self._repo.head.reference = self._repo.heads[branch] - self._repo.head.reset(index=True, working_tree=True) - - remote = self._find_or_add_remote() - self._update_from_remote(remote, branch) - - def _clone_repository(self) -> None: - kwargs = {} - if self.config.branch: - kwargs["branch"] = self.config.branch - - self._repo = Repo.clone_from( - self.config.repository_url, self.config.working_directory, **kwargs - ) - - def _find_or_add_remote(self): - remote = None - for r in self._repo.remotes: - if r.url == self.config.repository_url: - remote = r - break - - if remote is None: - remote = self._repo.create_remote( - "added_by_vulnerablecode", url=self.config.repository_url - ) - - return remote - - def _update_from_remote(self, remote, branch) -> None: - fetch_info = remote.fetch() - if len(fetch_info) == 0: - return - branch = self._repo.branches[branch] - branch.set_reference(remote.refs[branch.name]) - self._repo.head.reset(index=True, working_tree=True) - - -def _include_file( - path: str, - subdir: Optional[str] = None, - recursive: bool = False, - file_ext: Optional[str] = None, -) -> bool: - match = True - - if subdir: - if not subdir.endswith(os.path.sep): - subdir = f"{subdir}{os.path.sep}" - - match = match and path.startswith(subdir) + if recursive: + glob = "**/*" + else: + glob = "*" - if not recursive: - match = match and (os.path.sep not in path[len(subdir or "") :]) + if file_ext: + glob = f"{glob}.{file_ext}" - if file_ext: - match = match and path.endswith(f".{file_ext}") + return {p for p in path.glob(glob) if p.is_file()} - return match + def advisory_data(self) -> Iterable[AdvisoryData]: + """ + Return AdvisoryData objects corresponding to the data being imported + """ + raise NotImplementedError # TODO: Needs rewrite diff --git a/vulnerabilities/importers/gitlab.py b/vulnerabilities/importers/gitlab.py index 214c680cc..7c0aebf40 100644 --- a/vulnerabilities/importers/gitlab.py +++ b/vulnerabilities/importers/gitlab.py @@ -29,6 +29,8 @@ from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage +from vulnerabilities.importer import GitConfig +from vulnerabilities.importer import GitImporter from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference from vulnerabilities.importer import UnMergeablePackageError @@ -75,27 +77,23 @@ class ForkError(Exception): pass -class GitLabAPIImporter(Importer): +class GitLabAPIImporter(GitImporter): spdx_license_expression = "MIT" license_url = "https://gitlab.com/gitlab-org/advisories-community/-/blob/main/LICENSE" - gitlab_url = "git+https://gitlab.com/gitlab-org/advisories-community/" + config = GitConfig( + repo_url="git+https://gitlab.com/gitlab-org/advisories-community/", + ) + + def __init__(self): + super().__init__(config=self.config) + self.files = self.collect_files(recursive=True, file_ext="yml") def advisory_data(self) -> Iterable[AdvisoryData]: - try: - fork_directory = fork_and_get_dir(url=self.gitlab_url) - except Exception as e: - logger.error(f"Can't clone url {self.gitlab_url}") - raise ForkError(self.gitlab_url) from e - for root_dir in os.listdir(fork_directory): - # skip well known files and directories that contain no advisory data - if root_dir in ("ci", "CODEOWNERS", "README.md", "LICENSE", ".git"): - continue - if root_dir not in PURL_TYPE_BY_GITLAB_SCHEME: - logger.error(f"Unknown package type: {root_dir}") - continue - for root, _, files in os.walk(os.path.join(fork_directory, root_dir)): - for file in files: - yield parse_gitlab_advisory(file=os.path.join(root, file)) + for file in self.files: + # split a file name /tmp/tmpi1klhpmd/pypi/gradio/CVE-2021-43831.yml + # to ('/', 'tmp', 'tmpi1klhpmd', 'pypi', 'gradio', 'CVE-2021-43831.yml') + if file.parts[3] in PURL_TYPE_BY_GITLAB_SCHEME: + yield parse_gitlab_advisory(file) def get_purl(package_slug): From 577f2d56d9c1a7ecdc08d6112bf6774e8f4760fa Mon Sep 17 00:00:00 2001 From: ziad Date: Tue, 2 Aug 2022 22:46:33 +0200 Subject: [PATCH 2/8] remove git-config Signed-off-by: ziad --- vulnerabilities/importer.py | 17 ++++++----------- vulnerabilities/importers/gitlab.py | 6 +----- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/vulnerabilities/importer.py b/vulnerabilities/importer.py index 4826a6fb1..cd5c24182 100644 --- a/vulnerabilities/importer.py +++ b/vulnerabilities/importer.py @@ -313,28 +313,23 @@ def advisory_data(self) -> Iterable[AdvisoryData]: raise NotImplementedError -@dataclasses.dataclass -class GitConfig: - repo_url: str - save_working_directory: bool = True - - class ForkError(Exception): pass class GitImporter(Importer): - def __init__(self, config): + def __init__(self, repo_url, save_working_directory=True): super().__init__() - self.config = config + self.repo_url = repo_url + self.save_working_directory = save_working_directory try: - self.vcs_response = fetch_via_vcs(self.config.repo_url) + self.vcs_response = fetch_via_vcs(self.repo_url) except Exception as e: - logger.error(f"Can't clone url {self.config.repo_url} - {e}") + logger.error(f"Can't clone url {self.repo_url} - {e}") raise ForkError def __exit__(self): - if not self.config.save_working_directory: + if not self.save_working_directory: shutil.rmtree(self.vcs_response.dest_dir) def collect_files( diff --git a/vulnerabilities/importers/gitlab.py b/vulnerabilities/importers/gitlab.py index 7c0aebf40..99fd20d68 100644 --- a/vulnerabilities/importers/gitlab.py +++ b/vulnerabilities/importers/gitlab.py @@ -29,7 +29,6 @@ from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import GitConfig from vulnerabilities.importer import GitImporter from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference @@ -80,12 +79,9 @@ class ForkError(Exception): class GitLabAPIImporter(GitImporter): spdx_license_expression = "MIT" license_url = "https://gitlab.com/gitlab-org/advisories-community/-/blob/main/LICENSE" - config = GitConfig( - repo_url="git+https://gitlab.com/gitlab-org/advisories-community/", - ) def __init__(self): - super().__init__(config=self.config) + super().__init__(repo_url="git+https://gitlab.com/gitlab-org/advisories-community/") self.files = self.collect_files(recursive=True, file_ext="yml") def advisory_data(self) -> Iterable[AdvisoryData]: From 7c09515948416382ec91f9caddae293d7622567f Mon Sep 17 00:00:00 2001 From: ziad Date: Mon, 8 Aug 2022 14:10:20 +0200 Subject: [PATCH 3/8] remove unused library, remove a git-test Signed-off-by: ziad --- vulnerabilities/importer.py | 4 - vulnerabilities/importers/gitlab.py | 6 - vulnerabilities/tests/test_data_source.py | 256 +--------------------- 3 files changed, 4 insertions(+), 262 deletions(-) diff --git a/vulnerabilities/importer.py b/vulnerabilities/importer.py index cd5c24182..b8e2661dc 100644 --- a/vulnerabilities/importer.py +++ b/vulnerabilities/importer.py @@ -12,7 +12,6 @@ import logging import os import shutil -import tempfile import traceback import xml.etree.ElementTree as ET from pathlib import Path @@ -23,10 +22,7 @@ from typing import Set from typing import Tuple -from binaryornot.helpers import is_binary_string from fetchcode.vcs import fetch_via_vcs -from git import DiffIndex -from git import Repo from license_expression import Licensing from packageurl import PackageURL from univers.version_range import VersionRange diff --git a/vulnerabilities/importers/gitlab.py b/vulnerabilities/importers/gitlab.py index 99fd20d68..59e908b7e 100644 --- a/vulnerabilities/importers/gitlab.py +++ b/vulnerabilities/importers/gitlab.py @@ -8,7 +8,6 @@ # import logging -import os import traceback from datetime import datetime from typing import Iterable @@ -30,7 +29,6 @@ from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage from vulnerabilities.importer import GitImporter -from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference from vulnerabilities.importer import UnMergeablePackageError from vulnerabilities.improver import Improver @@ -72,10 +70,6 @@ def fork_and_get_dir(url): return fetch_via_vcs(url).dest_dir -class ForkError(Exception): - pass - - class GitLabAPIImporter(GitImporter): spdx_license_expression = "MIT" license_url = "https://gitlab.com/gitlab-org/advisories-community/-/blob/main/LICENSE" diff --git a/vulnerabilities/tests/test_data_source.py b/vulnerabilities/tests/test_data_source.py index fe17cafa7..8ba8503bf 100644 --- a/vulnerabilities/tests/test_data_source.py +++ b/vulnerabilities/tests/test_data_source.py @@ -7,24 +7,19 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -import datetime import os -import shutil -import tempfile import xml.etree.ElementTree as ET -import zipfile +from typing import Iterable from unittest import TestCase -from unittest.mock import MagicMock -from unittest.mock import patch -import git import pytest from packageurl import PackageURL +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.importer import ForkError from vulnerabilities.importer import GitImporter -from vulnerabilities.importer import InvalidConfigurationError +from vulnerabilities.importer import Importer from vulnerabilities.importer import OvalImporter -from vulnerabilities.importer import _include_file from vulnerabilities.oval_parser import OvalParser BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -41,249 +36,6 @@ def load_oval_data(): return etrees_of_oval -@pytest.fixture -def clone_url(tmp_path): - git_dir = tmp_path / "git_dir" - repo = git.Repo.init(str(git_dir)) - new_file_path = str(git_dir / "file") - open(new_file_path, "wb").close() - repo.index.add([new_file_path]) - repo.index.commit("Added a new file") - try: - yield str(git_dir) - finally: - shutil.rmtree(git_dir) - - -@pytest.fixture -def clone_url2(tmp_path): - git_dir = tmp_path / "git_dir2" - repo = git.Repo.init(str(git_dir)) - new_file_path = str(git_dir / "file2") - open(new_file_path, "wb").close() - repo.index.add([new_file_path]) - repo.index.commit("Added a new file") - - try: - yield str(git_dir) - finally: - shutil.rmtree(git_dir) - - -def mk_ds(**kwargs): - # just for convenience, since this is a mandatory parameter we always pass a value - if "repository_url" not in kwargs: - kwargs["repository_url"] = "asdf" - - last_run_date = kwargs.pop("last_run_date", None) - cutoff_date = kwargs.pop("cutoff_date", None) - - # batch_size is a required parameter of the base class, unrelated to these tests - return GitImporter( - batch_size=100, last_run_date=last_run_date, cutoff_date=cutoff_date, config=kwargs - ) - - -def test_GitImporter_repository_url_required(no_mkdir, no_rmtree): - - with pytest.raises(InvalidConfigurationError): - GitImporter(batch_size=100) - - -def test_GitImporter_validate_configuration_create_working_directory_must_be_set_when_working_directory_is_default( - no_mkdir, no_rmtree -): - - with pytest.raises(InvalidConfigurationError): - mk_ds(create_working_directory=False) - - -def test_GitImporter_validate_configuration_remove_working_directory_must_be_set_when_working_directory_is_default( - no_mkdir, no_rmtree -): - - with pytest.raises(InvalidConfigurationError): - mk_ds(remove_working_directory=False) - - -@patch("os.path.exists", return_value=True) -def test_GitImporter_validate_configuration_remove_working_directory_is_applied( - no_mkdir, no_rmtree -): - - ds = mk_ds(remove_working_directory=False, working_directory="/some/directory") - - assert not ds.config.remove_working_directory - - -def test_GitImporter_validate_configuration_working_directory_must_exist_when_create_working_directory_is_not_set( - no_mkdir, no_rmtree -): - - with pytest.raises(InvalidConfigurationError): - mk_ds(working_directory="/does/not/exist", create_working_directory=False) - - -def test_GitImporter_contextmgr_working_directory_is_created_and_removed(tmp_path, clone_url): - - wd = tmp_path / "working" - ds = mk_ds( - working_directory=str(wd), - create_working_directory=True, - remove_working_directory=True, - repository_url=clone_url, - ) - - with ds: - assert str(wd) == ds.config.working_directory - assert (wd / ".git").exists() - assert (wd / "file").exists() - - assert not (wd / ".git").exists() - - -@patch("tempfile.mkdtemp") -def test_GitImporter_contextmgr_calls_mkdtemp_if_working_directory_is_not_set( - mkdtemp, tmp_path, clone_url -): - - mkdtemp.return_value = str(tmp_path / "working") - ds = mk_ds(repository_url=clone_url) - - with ds: - assert mkdtemp.called - assert ds.config.working_directory == str(tmp_path / "working") - - -def test_GitImporter_contextmgr_uses_existing_repository( - clone_url, - clone_url2, - no_mkdir, - no_rmtree, -): - ds = mk_ds( - working_directory=clone_url, - repository_url=clone_url2, - create_working_directory=False, - remove_working_directory=False, - ) - - with ds: - # also make sure we switch the branch (original do not have file2) - assert os.path.exists(os.path.join(ds.config.working_directory, "file2")) - - assert os.path.exists(ds.config.working_directory) - - -def test__include_file(): - - assert _include_file("foo.json", subdir=None, recursive=False, file_ext=None) - assert not _include_file("foo/bar.json", subdir=None, recursive=False, file_ext=None) - assert _include_file("foo/bar.json", subdir="foo/", recursive=False, file_ext=None) - assert _include_file("foo/bar.json", subdir="foo", recursive=False, file_ext=None) - assert not _include_file("foobar.json", subdir="foo", recursive=False, file_ext=None) - assert _include_file("foo/bar.json", subdir=None, recursive=True, file_ext=None) - assert not _include_file("foo/bar.json", subdir=None, recursive=True, file_ext="yaml") - assert _include_file("foo/bar/baz.json", subdir="foo", recursive=True, file_ext="json") - assert not _include_file("bar/foo/baz.json", subdir="foo", recursive=True, file_ext="json") - - -class GitImporterTest(TestCase): - - tempdir = None - - @classmethod - def setUpClass(cls) -> None: - cls.tempdir = tempfile.mkdtemp() - zip_path = os.path.join(TEST_DATA, "advisory-db.zip") - - with zipfile.ZipFile(zip_path, "r") as zip_ref: - zip_ref.extractall(cls.tempdir) - - @classmethod - def tearDownClass(cls) -> None: - shutil.rmtree(cls.tempdir) - - def setUp(self) -> None: - self.repodir = os.path.join(self.tempdir, "advisory-db") - - def mk_ds(self, **kwargs) -> GitImporter: - kwargs["working_directory"] = self.repodir - kwargs["create_working_directory"] = False - kwargs["remove_working_directory"] = False - - ds = mk_ds(**kwargs) - ds._update_from_remote = MagicMock() - return ds - - def test_file_changes_last_run_date_and_cutoff_date_is_None(self): - - ds = self.mk_ds(last_run_date=None, cutoff_date=None) - - with ds: - added_files, updated_files = ds.file_changes( - subdir="rust", recursive=True, file_ext="toml" - ) - - assert len(updated_files) == 0 - - assert set(added_files) == { - os.path.join(self.repodir, f) - for f in { - "rust/cargo/CVE-2019-16760.toml", - "rust/rustdoc/CVE-2018-1000622.toml", - "rust/std/CVE-2018-1000657.toml", - "rust/std/CVE-2018-1000810.toml", - "rust/std/CVE-2019-12083.toml", - } - } - - def test_file_changes_cutoff_date_is_now(self): - - ds = self.mk_ds(last_run_date=None, cutoff_date=datetime.datetime.now()) - - with ds: - added_files, updated_files = ds.file_changes( - subdir="cargo", recursive=True, file_ext="toml" - ) - - assert len(added_files) == 0 - assert len(updated_files) == 0 - - def test_file_changes_include_new_advisories(self): - - last_run_date = datetime.datetime(year=2020, month=3, day=29) - cutoff_date = last_run_date - datetime.timedelta(weeks=52 * 3) - ds = self.mk_ds(last_run_date=last_run_date, cutoff_date=cutoff_date) - - with ds: - added_files, updated_files = ds.file_changes( - subdir="crates", recursive=True, file_ext="toml" - ) - - assert len(added_files) >= 2 - assert os.path.join(self.repodir, "crates/bitvec/RUSTSEC-2020-0007.toml") in added_files - assert os.path.join(self.repodir, "crates/hyper/RUSTSEC-2020-0008.toml") in added_files - assert len(updated_files) == 0 - - def test_file_changes_include_fixed_advisories(self): - # pick a date that includes commit 9889ed0831b4fb4beb7675de361926d2e9a99c20 - # ("Fix patched version for RUSTSEC-2020-0008") - last_run_date = datetime.datetime( - year=2020, month=3, day=31, hour=17, minute=40, tzinfo=datetime.timezone.utc - ) - ds = self.mk_ds(last_run_date=last_run_date, cutoff_date=None) - - with ds: - added_files, updated_files = ds.file_changes( - subdir="crates", recursive=True, file_ext="toml" - ) - - assert len(added_files) == 0 - assert len(updated_files) == 1 - assert os.path.join(self.repodir, "crates/hyper/RUSTSEC-2020-0008.toml") in updated_files - - class TestOvalImporter(TestCase): @classmethod def setUpClass(cls): From 637c3d5966d8cec4cb4f2e03fd2b08a899127004 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 9 Aug 2022 19:34:38 +0530 Subject: [PATCH 4/8] Refactor Git importer Signed-off-by: Tushar Goel --- vulnerabilities/importer.py | 52 +++++++------------------ vulnerabilities/importers/__init__.py | 2 +- vulnerabilities/importers/gitlab.py | 56 ++++++++++++++++++++++----- 3 files changed, 62 insertions(+), 48 deletions(-) diff --git a/vulnerabilities/importer.py b/vulnerabilities/importer.py index b8e2661dc..9edf63d00 100644 --- a/vulnerabilities/importer.py +++ b/vulnerabilities/importer.py @@ -314,48 +314,26 @@ class ForkError(Exception): class GitImporter(Importer): - def __init__(self, repo_url, save_working_directory=True): + def __init__(self, repo_url): super().__init__() self.repo_url = repo_url - self.save_working_directory = save_working_directory - try: - self.vcs_response = fetch_via_vcs(self.repo_url) - except Exception as e: - logger.error(f"Can't clone url {self.repo_url} - {e}") - raise ForkError - - def __exit__(self): - if not self.save_working_directory: - shutil.rmtree(self.vcs_response.dest_dir) - - def collect_files( - self, - subdir: str = None, - recursive: bool = False, - file_ext: Optional[str] = None, - ) -> Set[Path]: - """ - Returns Set of all file names - :param subdir: filter by files in this directory - :param recursive: whether to include files in subdirectories - :param file_ext: filter files by this extension - """ - if subdir is None: - working_dir = self.vcs_response.dest_dir - else: - working_dir = os.path.join(self.vcs_response.dest_dir, subdir) - - path = Path(working_dir) + self.vcs_response = None - if recursive: - glob = "**/*" - else: - glob = "*" + def __enter__(self): + super().__enter__() + self.clone() + return self - if file_ext: - glob = f"{glob}.{file_ext}" + def __exit__(self): + self.vcs_response.delete() - return {p for p in path.glob(glob) if p.is_file()} + def clone(self): + try: + self.vcs_response = fetch_via_vcs(self.repo_url) + except Exception as e: + msg = f"Failed to fetch {self.repo_url} via vcs: {e}" + logger.error(msg) + raise ForkError(msg) from e def advisory_data(self) -> Iterable[AdvisoryData]: """ diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index b3e6063f0..7c38680cc 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -26,7 +26,7 @@ redhat.RedhatImporter, pysec.PyPIImporter, debian.DebianImporter, - gitlab.GitLabAPIImporter, + gitlab.GitLabGitImporter, ] IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY} diff --git a/vulnerabilities/importers/gitlab.py b/vulnerabilities/importers/gitlab.py index 59e908b7e..0eadd3746 100644 --- a/vulnerabilities/importers/gitlab.py +++ b/vulnerabilities/importers/gitlab.py @@ -10,6 +10,7 @@ import logging import traceback from datetime import datetime +from pathlib import Path from typing import Iterable from typing import List from typing import Mapping @@ -70,20 +71,53 @@ def fork_and_get_dir(url): return fetch_via_vcs(url).dest_dir -class GitLabAPIImporter(GitImporter): +class GitLabGitImporter(GitImporter): spdx_license_expression = "MIT" license_url = "https://gitlab.com/gitlab-org/advisories-community/-/blob/main/LICENSE" def __init__(self): super().__init__(repo_url="git+https://gitlab.com/gitlab-org/advisories-community/") - self.files = self.collect_files(recursive=True, file_ext="yml") def advisory_data(self) -> Iterable[AdvisoryData]: - for file in self.files: - # split a file name /tmp/tmpi1klhpmd/pypi/gradio/CVE-2021-43831.yml - # to ('/', 'tmp', 'tmpi1klhpmd', 'pypi', 'gradio', 'CVE-2021-43831.yml') - if file.parts[3] in PURL_TYPE_BY_GITLAB_SCHEME: - yield parse_gitlab_advisory(file) + try: + self.clone() + path = Path(self.vcs_response.dest_dir) + + glob = "**/*.yml" + files = (p for p in path.glob(glob) if p.is_file()) + for file in files: + # split a path according to gitlab conventions where package type and name are a part of path + # For example with this path: + # /tmp/tmpi1klhpmd/pypi/gradio/CVE-2021-43831.yml + # the package type is pypi and the package name is gradio + # to ('/', 'tmp', 'tmpi1klhpmd', 'pypi', 'gradio', 'CVE-2021-43831.yml') + purl_type = get_gitlab_package_type(path=file) + if not purl_type: + logger.error(f"Unknow gitlab directory structure {file!r}") + continue + + if purl_type in PURL_TYPE_BY_GITLAB_SCHEME: + yield parse_gitlab_advisory(file) + + else: + logger.error(f"Unknow package type {purl_type!r}") + continue + finally: + if self.vcs_response: + self.vcs_response.delete() + + +def get_gitlab_package_type(path: Path): + """ + Return a package type extracted from a gitlab advisory path or None + """ + parts = path.parts[-3:] + + if len(parts) < 3: + return + + type, _name, _vid = parts + return type def get_purl(package_slug): @@ -156,10 +190,12 @@ def parse_gitlab_advisory(file): identifiers: - "GMS-2018-26" """ - with open(file, "r") as f: + with open(file) as f: gitlab_advisory = saneyaml.load(f) if not isinstance(gitlab_advisory, dict): - logger.error(f"parse_yaml_file: yaml_file is not of type `dict`: {gitlab_advisory!r}") + logger.error( + f"parse_gitlab_advisory: unknown gitlab advisory format in {file!r} with data: {gitlab_advisory!r}" + ) return # refer to schema here https://gitlab.com/gitlab-org/advisories-community/-/blob/main/ci/schema/schema.json @@ -249,7 +285,7 @@ def __init__(self) -> None: @property def interesting_advisories(self) -> QuerySet: - return Advisory.objects.filter(created_by=GitLabAPIImporter.qualified_name) + return Advisory.objects.filter(created_by=GitLabGitImporter.qualified_name) def get_package_versions( self, package_url: PackageURL, until: Optional[datetime] = None From dc605b00bd3009e82f31f25080f4939698229f39 Mon Sep 17 00:00:00 2001 From: ziad Date: Tue, 16 Aug 2022 15:30:33 +0200 Subject: [PATCH 5/8] add a test Signed-off-by: ziad --- vulnerabilities/importers/gitlab.py | 10 ++------ vulnerabilities/tests/test_gitlab.py | 35 ++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/vulnerabilities/importers/gitlab.py b/vulnerabilities/importers/gitlab.py index 0eadd3746..e970a4a7b 100644 --- a/vulnerabilities/importers/gitlab.py +++ b/vulnerabilities/importers/gitlab.py @@ -86,11 +86,6 @@ def advisory_data(self) -> Iterable[AdvisoryData]: glob = "**/*.yml" files = (p for p in path.glob(glob) if p.is_file()) for file in files: - # split a path according to gitlab conventions where package type and name are a part of path - # For example with this path: - # /tmp/tmpi1klhpmd/pypi/gradio/CVE-2021-43831.yml - # the package type is pypi and the package name is gradio - # to ('/', 'tmp', 'tmpi1klhpmd', 'pypi', 'gradio', 'CVE-2021-43831.yml') purl_type = get_gitlab_package_type(path=file) if not purl_type: logger.error(f"Unknow gitlab directory structure {file!r}") @@ -111,13 +106,12 @@ def get_gitlab_package_type(path: Path): """ Return a package type extracted from a gitlab advisory path or None """ - parts = path.parts[-3:] + parts = path.parts if len(parts) < 3: return - type, _name, _vid = parts - return type + return parts[3] def get_purl(package_slug): diff --git a/vulnerabilities/tests/test_gitlab.py b/vulnerabilities/tests/test_gitlab.py index bad3eae4f..0e2a05472 100644 --- a/vulnerabilities/tests/test_gitlab.py +++ b/vulnerabilities/tests/test_gitlab.py @@ -9,12 +9,16 @@ import json import os +from pathlib import Path from unittest import mock import pytest +from packageurl import PackageURL from vulnerabilities.importer import AdvisoryData from vulnerabilities.importers.gitlab import GitLabBasicImprover +from vulnerabilities.importers.gitlab import get_gitlab_package_type +from vulnerabilities.importers.gitlab import get_purl from vulnerabilities.importers.gitlab import parse_gitlab_advisory from vulnerabilities.improvers.default import DefaultImprover from vulnerabilities.tests import util_tests @@ -84,3 +88,34 @@ def test_gitlab_improver(mock_response, pkg_type): inference = [data.to_dict() for data in improver.get_inferences(advisory)] result.extend(inference) util_tests.check_results_against_json(result, expected_file) + + +def test_get_purl(): + assert get_purl("nuget/MessagePack") == PackageURL(type="nuget", name="MessagePack") + assert get_purl("nuget/Microsoft.NETCore.App") == PackageURL( + type="nuget", name="Microsoft.NETCore.App" + ) + assert get_purl("npm/fresh") == PackageURL(type="npm", name="fresh") + + +def test_get_gitlab_package_type(): + assert ( + get_gitlab_package_type(Path("/tmp/tmp9317bd5i/maven/com.google.gwt/gwt/CVE-2013-4204.yml")) + == "maven" + ) + assert ( + get_gitlab_package_type( + Path( + "/tmp/tmp9317bd5i/maven/io.projectreactor.netty/reactor-netty-http/CVE-2020-5404.yml" + ) + ) + == "maven" + ) + assert ( + get_gitlab_package_type( + Path("/tmp/tmp9317bd5i/go/github.com/cloudflare/cfrpki/CVE-2021-3909.yml") + ) + == "go" + ) + assert get_gitlab_package_type(Path("/tmp/tmp9317bd5i/gem/rexml/CVE-2021-28965.yml")) == "gem" + assert get_gitlab_package_type(Path()) is None From ec8820552fff8a0680df1c3f5a02dcdc5114ec22 Mon Sep 17 00:00:00 2001 From: ziad Date: Sat, 20 Aug 2022 16:57:30 +0200 Subject: [PATCH 6/8] rewrite get_gitlab_package_type Signed-off-by: ziad --- vulnerabilities/importers/gitlab.py | 22 ++++++++++------------ vulnerabilities/tests/test_gitlab.py | 19 ++++++++++++++----- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/vulnerabilities/importers/gitlab.py b/vulnerabilities/importers/gitlab.py index e970a4a7b..0016fb974 100644 --- a/vulnerabilities/importers/gitlab.py +++ b/vulnerabilities/importers/gitlab.py @@ -17,7 +17,6 @@ from typing import Optional import pytz -import saneyaml from dateutil import parser as dateparser from django.db.models.query import QuerySet from fetchcode.vcs import fetch_via_vcs @@ -42,6 +41,7 @@ from vulnerabilities.utils import AffectedPackage as LegacyAffectedPackage from vulnerabilities.utils import build_description from vulnerabilities.utils import get_affected_packages_by_patched_package +from vulnerabilities.utils import load_yaml from vulnerabilities.utils import nearest_patched_package from vulnerabilities.utils import resolve_version_range @@ -86,7 +86,7 @@ def advisory_data(self) -> Iterable[AdvisoryData]: glob = "**/*.yml" files = (p for p in path.glob(glob) if p.is_file()) for file in files: - purl_type = get_gitlab_package_type(path=file) + purl_type = get_gitlab_package_type(path=file, root=path) if not purl_type: logger.error(f"Unknow gitlab directory structure {file!r}") continue @@ -102,16 +102,14 @@ def advisory_data(self) -> Iterable[AdvisoryData]: self.vcs_response.delete() -def get_gitlab_package_type(path: Path): +def get_gitlab_package_type(path: Path, root: Path): """ - Return a package type extracted from a gitlab advisory path or None + Return a package type extracted from a gitlab advisory path """ - parts = path.parts - - if len(parts) < 3: - return - - return parts[3] + relative = path.relative_to(root) + parts = relative.parts + gitlab_schema = parts[0] + return gitlab_schema def get_purl(package_slug): @@ -184,8 +182,8 @@ def parse_gitlab_advisory(file): identifiers: - "GMS-2018-26" """ - with open(file) as f: - gitlab_advisory = saneyaml.load(f) + gitlab_advisory = load_yaml(file) + if not isinstance(gitlab_advisory, dict): logger.error( f"parse_gitlab_advisory: unknown gitlab advisory format in {file!r} with data: {gitlab_advisory!r}" diff --git a/vulnerabilities/tests/test_gitlab.py b/vulnerabilities/tests/test_gitlab.py index 0e2a05472..f96483eec 100644 --- a/vulnerabilities/tests/test_gitlab.py +++ b/vulnerabilities/tests/test_gitlab.py @@ -100,22 +100,31 @@ def test_get_purl(): def test_get_gitlab_package_type(): assert ( - get_gitlab_package_type(Path("/tmp/tmp9317bd5i/maven/com.google.gwt/gwt/CVE-2013-4204.yml")) + get_gitlab_package_type( + Path("/tmp/tmp9317bd5i/maven/com.google.gwt/gwt/CVE-2013-4204.yml"), + Path("/tmp/tmp9317bd5i/"), + ) == "maven" ) assert ( get_gitlab_package_type( Path( "/tmp/tmp9317bd5i/maven/io.projectreactor.netty/reactor-netty-http/CVE-2020-5404.yml" - ) + ), + Path("/tmp/tmp9317bd5i/"), ) == "maven" ) assert ( get_gitlab_package_type( - Path("/tmp/tmp9317bd5i/go/github.com/cloudflare/cfrpki/CVE-2021-3909.yml") + Path("/tmp/tmp9317bd5i/go/github.com/cloudflare/cfrpki/CVE-2021-3909.yml"), + Path("/tmp/tmp9317bd5i/"), ) == "go" ) - assert get_gitlab_package_type(Path("/tmp/tmp9317bd5i/gem/rexml/CVE-2021-28965.yml")) == "gem" - assert get_gitlab_package_type(Path()) is None + assert ( + get_gitlab_package_type( + Path("/tmp/tmp9317bd5i/gem/rexml/CVE-2021-28965.yml"), Path("/tmp/tmp9317bd5i/") + ) + == "gem" + ) From fdfa4fa0c49540d4912904f8f97b29e00f35bb1a Mon Sep 17 00:00:00 2001 From: ziad Date: Mon, 25 Jul 2022 12:56:50 +0200 Subject: [PATCH 7/8] npm importer - improver migration Signed-off-by: ziad --- vulnerabilities/importer.py | 31 ++++- vulnerabilities/importers/npm.py | 154 +++++++++++----------- vulnerabilities/tests/test_npm.py | 211 ++++++++++++++++-------------- vulnerabilities/utils.py | 23 +++- 4 files changed, 240 insertions(+), 179 deletions(-) diff --git a/vulnerabilities/importer.py b/vulnerabilities/importer.py index bcc711f9d..bfef86841 100644 --- a/vulnerabilities/importer.py +++ b/vulnerabilities/importer.py @@ -312,10 +312,28 @@ def advisory_data(self) -> Iterable[AdvisoryData]: raise NotImplementedError +@dataclasses.dataclass +class GitConfig: + repository_url: str + branch: Optional[str] = None + create_working_directory: bool = True + remove_working_directory: bool = True + working_directory: Optional[str] = "" + last_run_date: Optional[str] = None + cutoff_date: Optional[str] = None + + # TODO: Needs rewrite class GitImporter(Importer): - def validate_configuration(self) -> None: + def __init__(self, config, cutoff_timestamp): + super().__init__() + self.config = config + self.cutoff_timestamp = cutoff_timestamp + self._ensure_working_directory() + self._ensure_repository() + + def validate_configuration(self) -> None: if not self.config.create_working_directory and self.config.working_directory is None: self.error( '"create_working_directory" is not set but "working_directory" is set to ' @@ -336,10 +354,6 @@ def validate_configuration(self) -> None: "the default, which calls tempfile.mkdtemp()" ) - def __enter__(self): - self._ensure_working_directory() - self._ensure_repository() - def __exit__(self, exc_type, exc_val, exc_tb): if self.config.remove_working_directory: shutil.rmtree(self.config.working_directory) @@ -353,7 +367,6 @@ def file_changes( """ Returns all added and modified files since last_run_date or cutoff_date (whichever is more recent). - :param subdir: filter by files in this directory :param recursive: whether to include files in subdirectories :param file_ext: filter files by this extension @@ -477,6 +490,12 @@ def _update_from_remote(self, remote, branch) -> None: branch.set_reference(remote.refs[branch.name]) self._repo.head.reset(index=True, working_tree=True) + def advisory_data(self): + raise NotImplementedError + + def error(self, param): + pass + def _include_file( path: str, diff --git a/vulnerabilities/importers/npm.py b/vulnerabilities/importers/npm.py index 0b962255b..5054fd383 100644 --- a/vulnerabilities/importers/npm.py +++ b/vulnerabilities/importers/npm.py @@ -8,9 +8,10 @@ # # Author: Navonil Das (@NavonilDas) - -import asyncio +import logging +from typing import Iterable from typing import List +from typing import Optional from typing import Set from typing import Tuple from urllib.parse import quote @@ -18,65 +19,47 @@ import pytz from dateutil.parser import parse from packageurl import PackageURL -from univers.version_range import VersionRange +from univers.version_range import NpmVersionRange +from univers.versions import InvalidVersion from univers.versions import SemverVersion from vulnerabilities.importer import AdvisoryData +from vulnerabilities.importer import GitConfig from vulnerabilities.importer import GitImporter from vulnerabilities.importer import Reference from vulnerabilities.package_managers import NpmVersionAPI +from vulnerabilities.package_managers import PackageVersion from vulnerabilities.utils import load_json from vulnerabilities.utils import nearest_patched_package NPM_URL = "https://registry.npmjs.org{}" +logger = logging.getLogger(__name__) class NpmImporter(GitImporter): - def __enter__(self): - super(NpmImporter, self).__enter__() - if not getattr(self, "_added_files", None): - self._added_files, self._updated_files = self.file_changes( - recursive=True, file_ext="json", subdir="./vuln/npm" - ) - - self._versions = NpmVersionAPI() - self.set_api(self.collect_packages()) - - def updated_advisories(self) -> Set[AdvisoryData]: - files = self._updated_files.union(self._added_files) - advisories = [] - for f in files: - processed_data = self.process_file(f) - if processed_data: - advisories.extend(processed_data) - return self.batch_advisories(advisories) - - def set_api(self, packages): - asyncio.run(self._versions.load_api(packages)) - - def collect_packages(self): - packages = set() - files = self._updated_files.union(self._added_files) - for f in files: - data = load_json(f) - packages.add(data["module_name"].strip()) - - return packages - - @property - def versions(self): # quick hack to make it patchable - return self._versions - - def process_file(self, file) -> List[AdvisoryData]: + license_url = "https://github.com/nodejs/security-wg/blob/main/LICENSE.md" + spdx_license_expression = "MIT" + config = GitConfig( + repository_url="https://github.com/nodejs/security-wg.git", + working_directory="npm", + branch="main", + ) + cutoff_timestamp = 1 + + def __init__(self): + super().__init__(config=self.config, cutoff_timestamp=self.cutoff_timestamp) + self._added_files, self._updated_files = self.file_changes( + recursive=True, file_ext="json", subdir="vuln/npm" + ) + self.pkg_manager_api = NpmVersionAPI() - record = load_json(file) - advisories = [] + def parse_advisory_data(self, record) -> Optional[AdvisoryData]: package_name = record["module_name"].strip() publish_date = parse(record["updated_at"]) publish_date = publish_date.replace(tzinfo=pytz.UTC) - all_versions = self.versions.get(package_name, until=publish_date).valid_versions + all_versions = self.pkg_manager_api.fetch(package_name) aff_range = record.get("vulnerable_versions") if not aff_range: aff_range = "" @@ -84,8 +67,8 @@ def process_file(self, file) -> List[AdvisoryData]: if not fixed_range: fixed_range = "" - if aff_range == "*" or fixed_range == "*": - return [] + # if aff_range == "*" or fixed_range == "*": + # return None impacted_versions, resolved_versions = categorize_versions( all_versions, aff_range, fixed_range @@ -93,23 +76,28 @@ def process_file(self, file) -> List[AdvisoryData]: impacted_purls = _versions_to_purls(package_name, impacted_versions) resolved_purls = _versions_to_purls(package_name, resolved_versions) + vuln_reference = [ Reference( url=NPM_URL.format(f'/-/npm/v1/advisories/{record["id"]}'), reference_id=record["id"], ) ] + cve_id = record.get("cves") or [] + + return AdvisoryData( + aliases=cve_id, + summary=record.get("overview", ""), + affected_packages=nearest_patched_package(impacted_purls, resolved_purls), + references=vuln_reference, + date_published=publish_date, + ) - for cve_id in record.get("cves") or [""]: - advisories.append( - AdvisoryData( - summary=record.get("overview", ""), - vulnerability_id=cve_id, - affected_packages=nearest_patched_package(impacted_purls, resolved_purls), - references=vuln_reference, - ) - ) - return advisories + def advisory_data(self) -> Iterable[AdvisoryData]: + files = self._updated_files.union(self._added_files) + for file in files: + record = load_json(file) + yield self.parse_advisory_data(record) def _versions_to_purls(package_name, versions): @@ -150,10 +138,10 @@ def normalize_ranges(version_range_string): def categorize_versions( - all_versions: Set[str], + all_versions: Iterable[PackageVersion], affected_version_range: str, fixed_version_range: str, -) -> Tuple[Set[str], Set[str]]: +) -> Tuple[Set[SemverVersion], Set[SemverVersion]]: """ Seperate list of affected versions and unaffected versions from all versions using the ranges specified. @@ -168,31 +156,51 @@ def categorize_versions( fix_spec = [] if affected_version_range: - aff_specs = normalize_ranges(affected_version_range) - aff_spec = [ - VersionRange.from_scheme_version_spec_string("semver", spec) - for spec in aff_specs - if len(spec) >= 3 - ] + aff_spec = get_version_range(affected_version_range) if fixed_version_range: - fix_specs = normalize_ranges(fixed_version_range) - fix_spec = [ - VersionRange.from_scheme_version_spec_string("semver", spec) - for spec in fix_specs - if len(spec) >= 3 - ] + fix_spec = get_version_range(fixed_version_range) + aff_ver, fix_ver = set(), set() + # Unaffected version is that version which is in the fixed_version_range # or which is absent in the affected_version_range - for ver in all_versions: - ver_obj = SemverVersion(ver) + for ver in get_all_versions(all_versions): - if not any([ver_obj in spec for spec in aff_spec]) or any( - [ver_obj in spec for spec in fix_spec] - ): + if not any([ver in spec for spec in aff_spec]) or any([ver in spec for spec in fix_spec]): fix_ver.add(ver) else: aff_ver.add(ver) return aff_ver, fix_ver + + +def get_version_range(version_range) -> List[NpmVersionRange]: + fix_specs = normalize_ranges(version_range) + ver_range_objs = [] + for spec in fix_specs: + if len(spec) >= 3: + try: + ver_range_objs.append(NpmVersionRange.from_string(f"vers:npm/{spec}")) + except InvalidVersion: + logger.error(f"InvalidVersionRange {spec}") + + return ver_range_objs + + +def get_all_versions(all_versions) -> List[SemverVersion]: + """ + + Args: + all_versions: + + Returns: + + """ + ver_objs = [] + for ver in all_versions: + try: + ver_objs.append(SemverVersion(ver.value)) + except InvalidVersion: + logger.error(f"InvalidVersion {ver.value}") + return ver_objs diff --git a/vulnerabilities/tests/test_npm.py b/vulnerabilities/tests/test_npm.py index 1b36f10a2..eccbaaa04 100644 --- a/vulnerabilities/tests/test_npm.py +++ b/vulnerabilities/tests/test_npm.py @@ -15,113 +15,114 @@ from unittest.mock import patch from django.test import TestCase +from univers.versions import SemverVersion from vulnerabilities import models from vulnerabilities.import_runner import ImportRunner from vulnerabilities.importers.npm import categorize_versions +from vulnerabilities.importers.npm import normalize_ranges from vulnerabilities.package_managers import NpmVersionAPI -from vulnerabilities.package_managers import Version +from vulnerabilities.package_managers import PackageVersion BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data/") -MOCK_VERSION_API = NpmVersionAPI( - cache={ - "jquery": {Version("3.4.0"), Version("3.8.0")}, - "kerberos": {Version("0.5.8"), Version("1.2.0")}, - "@hapi/subtext": { - Version("3.7.0"), - Version("4.1.1"), - Version("6.1.3"), - Version("7.0.0"), - Version("7.0.5"), - }, - } -) - +# MOCK_VERSION_API = NpmVersionAPI( +# cache={ +# "jquery": {Version("3.4.0"), Version("3.8.0")}, +# "kerberos": {Version("0.5.8"), Version("1.2.0")}, +# "@hapi/subtext": { +# Version("3.7.0"), +# Version("4.1.1"), +# Version("6.1.3"), +# Version("7.0.0"), +# Version("7.0.5"), +# }, +# } +# ) -@patch("vulnerabilities.importers.NpmImporter._update_from_remote") +# +# @patch ( "vulnerabilities.importers.NpmImporter._update_from_remote" ) class NpmImportTest(TestCase): - tempdir = None - - @classmethod - def setUpClass(cls) -> None: - cls.tempdir = tempfile.mkdtemp() - zip_path = os.path.join(TEST_DATA, "npm.zip") - - with zipfile.ZipFile(zip_path, "r") as zip_ref: - zip_ref.extractall(cls.tempdir) - - cls.importer = models.Importer.objects.create( - name="npm_unittests", - license="", - last_run=None, - data_source="NpmImporter", - data_source_cfg={ - "repository_url": "https://example.git", - "working_directory": os.path.join(cls.tempdir, "npm/npm_test"), - "create_working_directory": False, - "remove_working_directory": False, - }, - ) - - @classmethod - def tearDownClass(cls) -> None: - # Make sure no requests for unexpected package names have been made during the tests. - shutil.rmtree(cls.tempdir) - assert len(MOCK_VERSION_API.cache) == 3, MOCK_VERSION_API.cache - - def test_import(self, _): - runner = ImportRunner(self.importer, 5) - - with patch("vulnerabilities.importers.NpmImporter.versions", new=MOCK_VERSION_API): - with patch("vulnerabilities.importers.NpmImporter.set_api"): - runner.run() - - assert models.Vulnerability.objects.count() == 3 - assert models.VulnerabilityReference.objects.count() == 3 - assert models.PackageRelatedVulnerability.objects.all().count() == 4 - - assert models.Package.objects.count() == 8 - - self.assert_for_package( - "jquery", {"3.4.0"}, {"3.8.0"}, "1518", vulnerability_id="CVE-2020-11022" - ) # nopep8 - self.assert_for_package("kerberos", {"0.5.8"}, {"1.2.0"}, "1514") - self.assert_for_package("subtext", {"4.1.1", "7.0.0"}, {"6.1.3", "7.0.5"}, "1476") - - def assert_for_package( - self, - package_name, - impacted_versions, - resolved_versions, - vuln_id, - vulnerability_id=None, - ): - vuln = None - - for version in impacted_versions: - pkg = models.Package.objects.get(name=package_name, version=version) - - assert pkg.vulnerabilities.count() == 1 - vuln = pkg.vulnerabilities.first() - if vulnerability_id: - assert vuln.vulnerability_id == vulnerability_id - - ref_url = f"https://registry.npmjs.org/-/npm/v1/advisories/{vuln_id}" - assert models.VulnerabilityReference.objects.get(url=ref_url, vulnerability=vuln) - - for version in resolved_versions: - pkg = models.Package.objects.get(name=package_name, version=version) - assert models.PackageRelatedVulnerability.objects.filter( - patched_package=pkg, vulnerability=vuln - ) + # + # @classmethod + # def setUpClass( cls ) -> None : + # cls.tempdir = tempfile.mkdtemp () + # zip_path = os.path.join ( TEST_DATA, "npm.zip" ) + # + # with zipfile.ZipFile ( zip_path, "r" ) as zip_ref : + # zip_ref.extractall ( cls.tempdir ) + # + # cls.importer = models.Importer.objects.create ( + # name="npm_unittests", + # license="", + # last_run=None, + # data_source="NpmImporter", + # data_source_cfg={ + # "repository_url": "https://example.git", + # "working_directory": os.path.join(cls.tempdir, "npm/npm_test"), + # "create_working_directory": False, + # "remove_working_directory": False, + # }, + # ) + + # @classmethod + # def tearDownClass( cls ) -> None : + # # Make sure no requests for unexpected package names have been made during the tests. + # shutil.rmtree ( cls.tempdir ) + # assert len ( MOCK_VERSION_API.cache ) == 3, MOCK_VERSION_API.cache + # + # def test_import( self, _ ) : + # runner = ImportRunner ( self.importer, 5 ) + # + # with patch ( "vulnerabilities.importers.NpmImporter.versions", new=MOCK_VERSION_API ) : + # with patch ( "vulnerabilities.importers.NpmImporter.set_api" ) : + # runner.run () + # + # assert models.Vulnerability.objects.count () == 3 + # assert models.VulnerabilityReference.objects.count () == 3 + # assert models.PackageRelatedVulnerability.objects.all ().count () == 4 + # + # assert models.Package.objects.count () == 8 + # + # self.assert_for_package ( + # "jquery", {"3.4.0"}, {"3.8.0"}, "1518", vulnerability_id="CVE-2020-11022" + # ) # nopep8 + # self.assert_for_package ( "kerberos", {"0.5.8"}, {"1.2.0"}, "1514" ) + # self.assert_for_package ( "subtext", {"4.1.1", "7.0.0"}, {"6.1.3", "7.0.5"}, "1476" ) + # + # def assert_for_package( + # self, + # package_name, + # impacted_versions, + # resolved_versions, + # vuln_id, + # vulnerability_id=None, + # ) : + # vuln = None + # + # for version in impacted_versions : + # pkg = models.Package.objects.get ( name=package_name, version=version ) + # + # assert pkg.vulnerabilities.count () == 1 + # vuln = pkg.vulnerabilities.first () + # if vulnerability_id : + # assert vuln.vulnerability_id == vulnerability_id + # + # ref_url = f"https://registry.npmjs.org/-/npm/v1/advisories/{vuln_id}" + # assert models.VulnerabilityReference.objects.get ( url=ref_url, vulnerability=vuln ) + # + # for version in resolved_versions : + # pkg = models.Package.objects.get ( name=package_name, version=version ) + # assert models.PackageRelatedVulnerability.objects.filter ( + # patched_package=pkg, vulnerability=vuln + # ) def test_categorize_versions_simple_ranges(): - all_versions = {"3.4.0", "3.8.0"} + all_versions = {PackageVersion("3.4.0"), PackageVersion("3.8.0")} impacted_ranges = "<3.5.0" resolved_ranges = ">=3.5.0" @@ -129,12 +130,18 @@ def test_categorize_versions_simple_ranges(): all_versions, impacted_ranges, resolved_ranges ) - assert impacted_versions == {"3.4.0"} - assert resolved_versions == {"3.8.0"} + assert impacted_versions == {SemverVersion("3.4.0")} + assert resolved_versions == {SemverVersion("3.8.0")} def test_categorize_versions_complex_ranges(): - all_versions = {"3.7.0", "4.1.1", "6.1.3", "7.0.0", "7.0.5"} + all_versions = { + PackageVersion("3.7.0"), + PackageVersion("4.1.1"), + PackageVersion("6.1.3"), + PackageVersion("7.0.0"), + PackageVersion("7.0.5"), + } impacted_ranges = ">=4.1.0 <6.1.3 || >= 7.0.0 <7.0.3" resolved_ranges = ">=6.1.3 <7.0.0 || >=7.0.3" @@ -142,5 +149,17 @@ def test_categorize_versions_complex_ranges(): all_versions, impacted_ranges, resolved_ranges ) - assert impacted_versions == {"4.1.1", "7.0.0"} - assert resolved_versions == {"3.7.0", "6.1.3", "7.0.5"} + assert impacted_versions == {SemverVersion("4.1.1"), SemverVersion("7.0.0")} + assert resolved_versions == { + SemverVersion("3.7.0"), + SemverVersion("6.1.3"), + SemverVersion("7.0.5"), + } + + +def test_normalize_ranges(): + assert normalize_ranges(">=6.1.3 < 7.0.0 || >=7.0.3") == [">=6.1.3,<7.0.0", ">=7.0.3"] + assert normalize_ranges(">=4.1.0 <6.1.3 || >= 7.0.0 <7.0.3") == [ + ">=4.1.0,<6.1.3", + ">=7.0.0,<7.0.3", + ] diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index 737b26171..6b3ae0808 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -29,6 +29,7 @@ from packageurl import PackageURL from univers.version_range import RANGE_CLASS_BY_SCHEMES from univers.version_range import VersionRange +from univers.versions import Version logger = logging.getLogger(__name__) @@ -39,8 +40,22 @@ @dataclasses.dataclass(order=True, frozen=True) class AffectedPackage: - vulnerable_package: PackageURL - patched_package: Optional[PackageURL] = None + package: PackageURL + affected_version_range: Optional[VersionRange] = None + fixed_version: Optional[Version] = None + + def to_dict(self): + """ + Return a serializable dict that can be converted back using self.from_dict + """ + affected_version_range = None + if self.affected_version_range: + affected_version_range = str(self.affected_version_range) + return { + "package": self.package.to_dict(), + "affected_version_range": affected_version_range, + "fixed_version": str(self.fixed_version) if self.fixed_version else None, + } def load_yaml(path): @@ -178,8 +193,8 @@ def nearest_patched_package( affected_package_with_patched_package_objects.append( AffectedPackage( - vulnerable_package=vulnerable_package.purl, - patched_package=patched_package.purl if patched_package else None, + package=vulnerable_package.purl, + fixed_version=patched_package.purl if patched_package else None, ) ) From a3bfb58759f6eb9e689535e2f0eb4c82be6fc9f6 Mon Sep 17 00:00:00 2001 From: ziadhany Date: Wed, 14 Sep 2022 21:23:49 +0200 Subject: [PATCH 8/8] fix npm affected package , add npm test Signed-off-by: ziadhany --- requirements.txt | 2 +- vulnerabilities/importers/__init__.py | 4 + vulnerabilities/importers/npm.py | 214 ++++++------------ vulnerabilities/tests/test_data/npm.zip | Bin 36323 -> 0 bytes .../tests/test_data/npm/npm-expected_1.json | 62 +++++ .../tests/test_data/npm/npm-expected_2.json | 64 ++++++ .../tests/test_data/npm/npm-expected_3.json | 158 +++++++++++++ .../tests/test_data/npm/npm_test_1.json | 32 +++ .../tests/test_data/npm/npm_test_2.json | 34 +++ .../tests/test_data/npm/npm_test_3.json | 32 +++ vulnerabilities/tests/test_npm.py | 174 +++----------- vulnerabilities/utils.py | 23 +- 12 files changed, 488 insertions(+), 311 deletions(-) delete mode 100644 vulnerabilities/tests/test_data/npm.zip create mode 100644 vulnerabilities/tests/test_data/npm/npm-expected_1.json create mode 100644 vulnerabilities/tests/test_data/npm/npm-expected_2.json create mode 100644 vulnerabilities/tests/test_data/npm/npm-expected_3.json create mode 100644 vulnerabilities/tests/test_data/npm/npm_test_1.json create mode 100644 vulnerabilities/tests/test_data/npm/npm_test_2.json create mode 100644 vulnerabilities/tests/test_data/npm/npm_test_3.json diff --git a/requirements.txt b/requirements.txt index d6e5e0119..fa8380d58 100644 --- a/requirements.txt +++ b/requirements.txt @@ -106,7 +106,7 @@ toml==0.10.2 tomli==2.0.1 traitlets==5.1.1 typing_extensions==4.1.1 -univers==30.7.0 +univers==30.8.0 urllib3==1.26.9 wcwidth==0.2.5 websocket-client==0.59.0 diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 7c38680cc..c7e189f01 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -12,8 +12,10 @@ from vulnerabilities.importers import github from vulnerabilities.importers import gitlab from vulnerabilities.importers import nginx +from vulnerabilities.importers import npm from vulnerabilities.importers import nvd from vulnerabilities.importers import openssl +from vulnerabilities.importers import pypa from vulnerabilities.importers import pysec from vulnerabilities.importers import redhat @@ -27,6 +29,8 @@ pysec.PyPIImporter, debian.DebianImporter, gitlab.GitLabGitImporter, + pypa.PyPaImporter, + npm.NpmImporter, ] IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY} diff --git a/vulnerabilities/importers/npm.py b/vulnerabilities/importers/npm.py index 5054fd383..af4fd7113 100644 --- a/vulnerabilities/importers/npm.py +++ b/vulnerabilities/importers/npm.py @@ -9,11 +9,10 @@ # Author: Navonil Das (@NavonilDas) import logging +from pathlib import Path from typing import Iterable from typing import List from typing import Optional -from typing import Set -from typing import Tuple from urllib.parse import quote import pytz @@ -24,13 +23,11 @@ from univers.versions import SemverVersion from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import GitConfig +from vulnerabilities.importer import AffectedPackage from vulnerabilities.importer import GitImporter from vulnerabilities.importer import Reference from vulnerabilities.package_managers import NpmVersionAPI -from vulnerabilities.package_managers import PackageVersion from vulnerabilities.utils import load_json -from vulnerabilities.utils import nearest_patched_package NPM_URL = "https://registry.npmjs.org{}" logger = logging.getLogger(__name__) @@ -39,163 +36,73 @@ class NpmImporter(GitImporter): license_url = "https://github.com/nodejs/security-wg/blob/main/LICENSE.md" spdx_license_expression = "MIT" - config = GitConfig( - repository_url="https://github.com/nodejs/security-wg.git", - working_directory="npm", - branch="main", - ) - cutoff_timestamp = 1 def __init__(self): - super().__init__(config=self.config, cutoff_timestamp=self.cutoff_timestamp) - self._added_files, self._updated_files = self.file_changes( - recursive=True, file_ext="json", subdir="vuln/npm" - ) - self.pkg_manager_api = NpmVersionAPI() - - def parse_advisory_data(self, record) -> Optional[AdvisoryData]: - package_name = record["module_name"].strip() - - publish_date = parse(record["updated_at"]) - publish_date = publish_date.replace(tzinfo=pytz.UTC) - - all_versions = self.pkg_manager_api.fetch(package_name) - aff_range = record.get("vulnerable_versions") - if not aff_range: - aff_range = "" - fixed_range = record.get("patched_versions") - if not fixed_range: - fixed_range = "" - - # if aff_range == "*" or fixed_range == "*": - # return None - - impacted_versions, resolved_versions = categorize_versions( - all_versions, aff_range, fixed_range - ) - - impacted_purls = _versions_to_purls(package_name, impacted_versions) - resolved_purls = _versions_to_purls(package_name, resolved_versions) - - vuln_reference = [ - Reference( - url=NPM_URL.format(f'/-/npm/v1/advisories/{record["id"]}'), - reference_id=record["id"], - ) - ] - cve_id = record.get("cves") or [] - - return AdvisoryData( - aliases=cve_id, - summary=record.get("overview", ""), - affected_packages=nearest_patched_package(impacted_purls, resolved_purls), - references=vuln_reference, - date_published=publish_date, - ) + super().__init__(repo_url="git+https://github.com/nodejs/security-wg.git") def advisory_data(self) -> Iterable[AdvisoryData]: - files = self._updated_files.union(self._added_files) + self.clone() + path = Path(self.vcs_response.dest_dir) + + glob = "vuln/npm/**/*.json" # subdir="vuln/npm" + files = (p for p in path.glob(glob) if p.is_file()) for file in files: + print(file) record = load_json(file) - yield self.parse_advisory_data(record) - - -def _versions_to_purls(package_name, versions): - purls = {f"pkg:npm/{quote(package_name)}@{v}" for v in versions} - return [PackageURL.from_string(s) for s in purls] - - -def normalize_ranges(version_range_string): - """ - - Splits version range strings with "||" operator into separate ranges. - - Removes spaces between range operator and range operands - - Normalizes 'x' ranges - Example: - >>> z = normalize_ranges(">=6.1.3 < 7.0.0 || >=7.0.3") - >>> assert z == [">=6.1.3,<7.0.0", ">=7.0.3"] - """ - - version_ranges = version_range_string.split("||") - version_ranges = list(map(str.strip, version_ranges)) - for id, version_range in enumerate(version_ranges): - - # TODO: This is cryptic, simplify this if possible - version_ranges[id] = ",".join(version_range.split()) - version_ranges[id] = version_ranges[id].replace(">=,", ">=") - version_ranges[id] = version_ranges[id].replace("<=,", "<=") - version_ranges[id] = version_ranges[id].replace("<=,", "<=") - version_ranges[id] = version_ranges[id].replace("<,", "<") - version_ranges[id] = version_ranges[id].replace(">,", ">") - - # "x" is interpretted as wild card character here. These are not part of semver - # spec. We replace the "x" with aribitarily large number to simulate the effect. - if ".x." in version_ranges[id]: - version_ranges[id] = version_ranges[id].replace(".x", ".10000.0") - if ".x" in version_ranges[id]: - version_ranges[id] = version_ranges[id].replace(".x", ".10000") - - return version_ranges - - -def categorize_versions( - all_versions: Iterable[PackageVersion], - affected_version_range: str, - fixed_version_range: str, -) -> Tuple[Set[SemverVersion], Set[SemverVersion]]: - """ - Seperate list of affected versions and unaffected versions from all versions - using the ranges specified. - - :return: impacted, resolved versions - """ - if not all_versions: - # NPM registry has no data regarding this package, we skip these - return set(), set() - - aff_spec = [] - fix_spec = [] - - if affected_version_range: - aff_spec = get_version_range(affected_version_range) - - if fixed_version_range: - fix_spec = get_version_range(fixed_version_range) - - aff_ver, fix_ver = set(), set() + yield parse_advisory_data(record) - # Unaffected version is that version which is in the fixed_version_range - # or which is absent in the affected_version_range - for ver in get_all_versions(all_versions): - if not any([ver in spec for spec in aff_spec]) or any([ver in spec for spec in fix_spec]): - fix_ver.add(ver) - else: - aff_ver.add(ver) +def parse_advisory_data(record) -> Optional[AdvisoryData]: + cves = record.get("cves") or [] + overview = record.get("overview", "") + package_name = record["module_name"].strip() - return aff_ver, fix_ver + publish_date = parse(record["updated_at"]) + publish_date = publish_date.replace(tzinfo=pytz.UTC) + pkg_manager_api = NpmVersionAPI() + all_versions = pkg_manager_api.fetch(package_name) + aff_range = record.get("vulnerable_versions") or "" + fixed_range = record.get("patched_versions") or "" -def get_version_range(version_range) -> List[NpmVersionRange]: - fix_specs = normalize_ranges(version_range) - ver_range_objs = [] - for spec in fix_specs: - if len(spec) >= 3: - try: - ver_range_objs.append(NpmVersionRange.from_string(f"vers:npm/{spec}")) - except InvalidVersion: - logger.error(f"InvalidVersionRange {spec}") + fixed_versions = get_fixed_version( + map_all_versions(all_versions), NpmVersionRange.from_native(fixed_range) + ) + # if aff_range == "*" or fixed_range == "*": + # return None - return ver_range_objs + vuln_reference = [ + Reference( + url=NPM_URL.format(f'/-/npm/v1/advisories/{record["id"]}'), + reference_id=record["id"], + ) + ] + + return AdvisoryData( + aliases=cves, + summary=overview, + affected_packages=[ + AffectedPackage( + package=PackageURL.from_string(f"pkg:npm/{quote(package_name)}"), + affected_version_range=NpmVersionRange.from_native(aff_range), + fixed_version=fixed_version, + ) + for fixed_version in fixed_versions + ], + references=vuln_reference, + date_published=publish_date, + ) -def get_all_versions(all_versions) -> List[SemverVersion]: +def map_all_versions(all_versions) -> List[SemverVersion]: """ + map all versions from PackageVersion to SemverVersion - Args: - all_versions: + Parameters: + all_versions (PackageVersion): List of PackageVersion Returns: - + List[SemverVersion]: return a list of SemverVersion """ ver_objs = [] for ver in all_versions: @@ -204,3 +111,22 @@ def get_all_versions(all_versions) -> List[SemverVersion]: except InvalidVersion: logger.error(f"InvalidVersion {ver.value}") return ver_objs + + +def get_fixed_version( + all_versions: List[SemverVersion], aff_range: NpmVersionRange +) -> List[SemverVersion]: + """ + return a list of SemverVersion fixed versions + """ + try: + fixed_versions = [] + if not all_versions or not all_versions: + return fixed_versions + for v in all_versions: + if v in aff_range: + fixed_versions.append(v) + return fixed_versions + except Exception as e: + logger.error(e) + return [] diff --git a/vulnerabilities/tests/test_data/npm.zip b/vulnerabilities/tests/test_data/npm.zip deleted file mode 100644 index fdb957a32e9bb63b4bc8b6315e5722cd2435d88c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36323 zcmeFZbyU>b{sv5gG)Q-M4KOstAvO2lMbREwEqxw_pDC0|AB##=*&+Q%e&A z23fVV)%4#lcP}&;ScE-z7?|IFsQw4R$l+Feos8#`A}Tfg$`Up0$HH#Ot^9K7jrE z^%qdrujnoorlzKFW;=m*=lO4+6Q6RW`1u7SXFp|%_RDsDqoNlM*^;P37Np2AMHl9enGn}B#{m#H*e^*)KU zu*%zOnx#Y>#!cU8hrnjBM_yKB8fc>5-W^sR-%X1thz*Z5({V0(1U zd;W#O)-95rZ1DRX{j4NX+^qxDJb$9M_~=dMjt$R)?ceF0Sq#l_S{`40YX#5NAX+0T z8jH*t&JEX_+zycRx06(3sBM#}I2^uRS&)Yx(s>^%GmIXNd}j%bIr)c02!pYz#6S~hF#42%ah9s7-! z%YdR~Lq$RQo!)RF^SW^UBhQg%bxeo^Bu^^WV1)xk2lj|1D5z=Pk>kuCb9L=7As8qK z5R9}1WbdG8hcM_NdqJC4VHIjT^*B(3;@tG`5##qA_OXso6O8s>9 z6uy47wr1_tP(zOO`_2vFPhl#-1w>KPbCY3I&*mEPmT9=+tR6L^-{;$fz?KtW)>ZD3 zFp5@|sgTY!JEE?}M8O+c$N!+pB?hHCwFNPe6U*6p7qOrqPPxl?%Am?J(wRR}!5-Fg zoDf=h4kqBVjv)TrXIl9}!$24p_LaQ9YFvAnCH% zoR_WGf}g(K3W+wqnIQ5>uDAuV#^}wTS@NQml$*47@7#O8?Y=RR?#Nkth_?Oj!`PS5 zNPCE>pGO-qIt&cpZ=%h@)!xy;+Rf30-SeRc*n=GYVGW4aUUDD>V*4btn^p5uu}0T= zT&tIw0sUO9!#eq0tWkR-#E;Z*H}RVoni3FYuP!pEJ+~FrYcC{o=q=w^j*pVL9IELH zrz*!Ovs#ZhG0@ea51Vj#o-g;YN5ftd!*hB9EoN(nQNgC= z;^A0eUTq=|YipiL_wHHHB-dK9w@DM5QXx&^V}Z98!xM)?a>;HBN%=ndX74I{oDJzA z;_lKx-?%TI`U~iD$nW_R6sT$h-4(Iqle#xlUV>Il)goCo$UG2@6b|F2YGXfsP&gXF zKRTC05jSS1TaO`F6i6mD5(rEtX~ABtX$ZjK$+7I}nQbp_LtB68$B7Fe#)-~EQ*m`> zwx*%CY!%;&8>hdZ$8CudN)yU?ys;ZePYoVKJ2PcAu^wrOV}Ve8vyf_rc+lMx>F!QxJ3~4nc9#v`;pB&0hP7CckiAv z{?j4Nu`ikNxkUyT?hJ0{MXk3H&%FtqQET!Pej;{$?97+1r{W~~=WYx*S8Hf!pa zr*W7`Fyd>bKWYcl?6_uI;)=tW!60#Dp-?g(9o4_`A(>unY+=wdV}L19jfsg`oYR7^ z>CmaW^k@%Oj}?5}nIld2KUQmy{H|yW3rFZhlSGmihc=cAB&i~fJmjueTEBbiz44Y5 zME3mJ<=M?zQG*lf$6Uv?_eO&{HTlzRPgjg&=`Q8>O)|_EZ1=Ge`;=eZ<_>xC8oYRs zl2NdKWLMVu9^Wf)NhQroaO9Mzx(hw>2-Se9%pgpE_pGKK#cob|Xm3>r^T1=iK^;*9 z!*+apf3s(8tzAI)$9K@vu@j&y{DFPi8xw1RSqt3_sg8ql+B&AE?^V&K3yYo}5)#{s zy+FBNHgI&aT2wWbql|RU&U>tvLeCDtszo8(!9$f0vBRW&sYoG#6)b8iO?~kZL83a& z=_ApLZxtR2E|;*L4iF+&Kwn(teMB(7CP{BKOm5l$ObMy z>zsd`FjA<^(1^GLiG*7d3Ey^v=JCu-q5SUl6xx^N{g;t3ap+7VZ$%kjDx3nfKYm9Rch-bFPe_flR_;@K{s_l9aR3L$MxXeoF&<6wWB!n|{L+3> zM*Wy)<&Fai``oGnaiDdTDN9a9mf?G)(I2oH%0#=QFsJ^w3|k$_>euRV#ce6osVGlA)uDBSXs{QPh zCHb4F+uob4Et6Ii<$3RwPh9g0x)@kr2#6?VeXkF{WWD}C=wcZQ$EwJ<2cFG@U&I(| zwts{CaatezQS2q{p|n2mvH-SkJT7reOXLMQahIm4rc*}*#pP8**Cxr9EP?gpLc!eX zE3dDoE5ZsHgorh(ElYz>Fx#5|Mz^6u&Z0ILAt*_~l0ST$<5!pUx=w!>=GhS}ZfQ-{ zj|F*7oQS@y=ezfG{2>!{VVxM-=k%)C^<9_?v-a_Y4~f=<)L0rzxTP|gr=|ZZFo_2u z|KrebXT&-VY7=e@bxbEIr&QO zXAJZ>;on|uEq6XT>DcMgJ5`FfEWpb`n=a(gQVUNSvC>$fd$NK73Cnf5KkX*3!EDh% zX&za*$u(X+Xby(fTSh`SM^6zRdQZbfqS&_sep7tYeCv8)UbNSiAE~97CSHs+d=q=< z8pQ~hkqwA2Fb}=q=Ut->s#010rfWDky1KEuJDG#rApfvku#c(X`g5TpOx|ah1)2Eq zcp-CQQ^jLWyTz=pAC>Lx9*?)$bM_>inkrgJh8rMJ*2Xh}ar8j31)rS6)j0CdC9MjmQZwtwWU_BHQLbxW^_R&G3nQJB+2!!9?y6_=8Zph;N%em z`k=i05-96(TRg6NsFdgu_XYNWffV%1&M@d2AJ+ME2Hr!v>fh;RCl?6&AKLjJlAslB z@SvNe?`y&og^Hg>Yu{>1_&=%i8=akzdd8skc^|%a_3Jm-PnV0L0~enhR{EXCy4XhH z9BKFVG>B2Ccd?vwz&LXK88)9)1Q0kw(AReM(;5Of3h|_^N?u_s*^lD>#u8ofF_voJ^4-9=ljxhL$k5?++&Nt?H)r6YmuAwd<#co>hq z&?mwb4-Ac?y96_0yF{kvi_VpEeiwDh=Cu2DCujG38v9Eh3VJOf(OFk`?W}yV_WKLI zIw*Lj+PGx6&3|KW!P}LZIImJTy6ImBbgT~%9n1nTGt8!oWZPJrpKi<>G*BPOc^Z8& zsuTj$k>##(Euae01<1g{62_h|&B=zGjecEtEZm0Frx_4Rm88~L;c=^>*L22v1XuE| zvz5tCx|`{Xvb&HJL%xif?x-@4;yEmP>_27ZiZ!|Kfn2}ay)83+?_N6RB+Y{HuJBYx8iMxaW$E7f z4cnVaag$ro!{*z@7cWQIf?V?kt*P2)CS9LIt7Pv!e@D3y5t&~H{yN<!x9pPu@G{l&a(Sd8O;CO7s5~jS5W@H$G zEc(5-aC){!xjl!E+fk1NX^pl;hE#^h3X36sr<3P}?F=N2uAB0<{UUmC6a`g%7oKRo zKP*7R=Nm=}$^OnY_jFDawTd^Y4o}#J_=U*irH{tQqHvwx8HjTGqqAYAHP3U_&BF%p8ilftJ=IWj;;y~(AYe** z#DJg>JHa^97mKu!vL1mQcNRvRHLtJbX$TpDJA+=f91Tv7;2gsJ_g=YvkA)rWF$m#} zZB=vhwL<`PHZ8W-TN%{zhQD^^pBIxn*Fv;JON(^_+_X*C6El9Fw}+%}5bi%mdVX+M z)TDra|62{lCAVY=h3;D(vi#@yp9QT9oG`ynk^eIT{%vsk`|RQ&|Lq|zmjA~p;1i{y z63B(!cXEfP!sU(I=*^Ro)JAe#(D{@!+g>PO=gb#wYsM z{4IIO>t?rbmkW%3QZF?wHt`895i>w7jjPAx*-;~(q^;i1ZOo1HVCvZKsc5c$ z>Z6vL7b!zS+#s=9F$oDj6k$?K#SP}w6pN$PEiEMhqC&>D=Yk*ccNWBfhgVUpk!sTA zmfj#3ILMHLOfEhI6W@Ih z>7OI!zcbO$MnEJi5;?wHUDcu9(}AuE*G2? zcR*Lr5zIsbIE21c6An*%(O(426#F5ZkR#8SX@0F%wc)eQ86!V;Xk@evr?=DK6V9*q zuI^KXaSGe{a7PFRIRf!pAP2=;fyiH*8m^KTgnPMKeAGD`zclF66RdKA+QX> zR&i7Y7T#YmBIsG^Mwe1E-^nlb<3A!5j^`hsKMGq6O?Fr(#S1sGf+f{ewg1jxkvAB& zV24RLXyacoHAyx--jtO6sFh^qGAsD%F$}fgCQN=}z52lZ8x2lB2H)!FfIYVnj?YDt z$GUH@9h&ubs0+~uX(om;Unhj!B08x!)V+vYwR-o>=rp&p27Wdl#~<_5%vCY!(Sk<@ zuO?vv!FM?IOCf=4@vpL_nTq%gvDA1PY%U&OM^Q@S0>Tq5Ra-sV%CLsNs0z?@e81{y`G$s_ zb>9qeaA&jqq$z4D*3lxoG%Z={8{G?u-QvYPyQXU1x3auqBDcPyukMxa-;X?-61-zW zRb!F$Zi<<$s7dB2cPlbiY+6L`WmQFe#a#2UX`F4eKs`QIEKi$l>4QacT{(b> z;5o-*+2gGOmWjC(dQFLN3H_omo&i3AoT1*S+4w5`uTn$n1CNwc|OeoRV@4(V_2-_$ z(P-=@R7Y-zV`!bzOpKzwFs;;j!4r}5<#S-24BINJ=ss5DwOT?8Tdw_cYiIJ%FAH zk*=~%uCSbSi6mW))vP{VFqbzxhv-8WUST#?!h3M%{{H8ErUPP8(-CD+zZOYf2=1ow)uPp%T%`QJeJ0!kpe+`LR6mq=ubD6IDV$O@r9o_3@Vj z0TIs87~0#~Y+aDu7p6_Uk6vs3Lq&GY-8GjGu@GY4C4ditU(ox;m)>m)!5b|-Ls(`p z#&`BoJOw8hfF|XPnW>PI;~~xG7Qr=_+jeB5!p$xfTgN`9|@yY|0aRMj*=IMczp!S$g#k3s#-OcHz?WG2e zu+`VbMhsTwM*%l`IPRGwh_X7=fsyEI0rmIiEez+Bm#GC`ozL&TZ{CChcRzBPm`JI4 zGPR_4hiG!xWJThf(6|M@=0;HXN{6H9rzaXmOo&aNoKPiAibz*o-PA*I zO5e8BPD+b}JyN2l)ANBy8^b+IB^sYKw#YVNnOSjPt>ZmUoV)ND_o^M0ZVmn@XV5s$ zadT%(V>}WmP146GwlFo7$xd79fxaUq<=3f^$&beqBb>SV@@L$!M&(O*2m{sor-`zG z<$VZPs2@fNaWTmIJ9`b7Sudo3;aJIU{Wf|oNUt+U35-mP5~ZTyi9|eGakpb&b6XWl ztHh_~gX51lB&?p8JrReA40YFbmUYYOK423Q42kp;OYdcQ#43@?-&@+-P3nwf9(~|l zJVBWA74W!;vZz5s{9P>fSVA2ES{5Z|ZkI|0M+K*GTly8SGTvth{bFMe*kLBA4u4pM z7PQ9rv75cp&7hcX{fP+4l$#JeuRnR7^MD&@i{{{)f+LsucS^j;7|D8rS1$vdwk_?9 zAVUPA|U4c@OHW? zKX-a3r$Lp@v#ZBiL?kW68I9gIx64y~M3w$j2-Z&Fv8o%MX5|tvXXDmZ%^h}SC8NG> z1Ht4}KFGc5arJqoDcWYULJr+oRdw7}DUuH%0)a-JK~l-rh=+(Bax( znQyQ5!z=@rH_Y)cHNCc-_Y9Oshxan4DTs!j8>1YC(`YhPt{wWxDlx=%+#*d1N9xUg zOSz9KPY$C*aXR%BxAYFR5T3Z0bjvsezNtM*G!`jHb8kP|y`$~sSm{po`th23a%}IA z#**XUUJ+*LDZi^|*m%E_AM5RM`V0j>&f4&VVVfy1kM|~!fl?vn6+BA(CUAZ-mTUrT zAl4$kL^tEb1)3J2S8DcHL7>B^?^pKw=uZm0mj@OxLQi&;Sv+5_tPZO7RF9VF9=jQJ zNM<5GU!sI@lG{4FPd@Rg?(Ht?Q-nMa$D}JTxSROJfu*hLr-AmHCk57UCJsSrH9zhiJ*bMn z&$?=B$S^Pu+TrJ_f(oiR{&vy^-Q&1Gzz}PXe<~~7RY#P2xUfYAOaP5IK2HsX-?7Mr zz<-<8rfH+ra%g_1^jJ7-1zF-&*kd>s#d%@wg=Ig_k`QTm9EJXnerb>}hEPZ~3(9m7 zhMjiFa2Ii$ndH2_|B#w}Szu8b_adeUYy3B*n&Ac=RR#>bkeu&9Z(_T=R(O!iI+8fyxXG&lWgik_~94T zQ;*OoSR;Kme(Xxy_T+?kv)iJViX_FbL1LlIBc_-nlT>$xp3;l+D(x$&B*B zfib<`6Xc24iC7qykRx*Dqvh2n2(UHUQieiEUL&2V#u#v#K z*BxG3i6}wp4W{bsHfY-7WmP*QT?8ma#Vnv~P1g65MD5*4m4&TrfPigyDb{9Eim-L# z`R-i03T(~e3Rg8D7KiNh{&ehih&{1vbV;@5rQZ~Y8{H-_#x@0p+79i2l&g4(xgVy6 z=JQia_gq^DcA#^rsf>n4b{mU?GF%goYBz@$gp8>!e-WbZSGCM@;q9t#r=q**cZ%(!2auo5P77_OhuS|*GY_NCSZ&<0N|B(wDc9k9 z^OPueC{Jzw%hIh~BQw{M+t96EF#5-Zf+dj=Cbp!o_};fgB9^mN>T;9=JR9w~%!yLq zQYJ<9Ei@@=$-&_Elx|ka5T|@Ry#(sQ@5-E<91Kul_3h(gz!Wnq0vCC(0$^y#kD z@o#5CQ;nDPGx8+4clVVtMsSb6#;n27KDpD6O`VieSR&ILa=-0(7AsIN(A;J|8T|fI zLVC+4Ug-OdwkpqoAUlqA=Z)Asc@{dl+Auws_ zM(hlf6Yb;cp$E_4=)NhrJP`}QTAM)r3gUdO{o0dDOyQv2VBY!BF6SK&drgWl+BmX@Jb&&-?wER8O<9B$i-#J0qQMxQj;Amo?Wh(Gi*r}iUaqi zW#w;K4a&Kdi`1j?HVsSBiKT0v^+nYtJX3%qZMbsVWit=W#WeDY5W3)q_ppy}lZ~UI z5_YSOgE=z3d-Z)9i01Fk>pksz@d(E!Xhy1r+%(MvoG68Uo$7E68Z9MYJ#)sVVRRRLu=9EEFJV^3%&!i+Rg= zB5R)RwAF7d($XmGP#?`S9(>z+nP1Dr!by{piJEd2SjGdMKwD&50Afd^=N436qOV|NCu%k17A$-wM-j)z zjp0oQdQCEP_4q>~=r#A(L&|Q2FIWjq@NUdel<@B+GkJf=fKC!sF0ni_eX;gO>ZIuc zFrzf$#?r%O+Y=}h^*)#9v43eqqD9@``CN^gcd2Upp3gpsm$o5nZt=s(i#iD}g@Ey- zxi!3kRY9NE2EcD!HvvJOI~J-dx2Ok(2%#u84sq{LtV?g-B(C~gDqJiZ+Sae@$d(?H zmYuE($+tK^^FFb+4PRmA&@X;jIDqc|q&_pIe2pBz-k;)~gNgmT%vaF~E{+8MrEw<( zCvRs{_}xL-&|!HNr2lI15OIk7Q_e=cj2Acj_)lU;JrK#F){?e3|TN=7q&Eh zU}pYySF5((M)Z$-Y#PXLa1Zs}PTv91ZLeJNh0p7jQwNIz_6=eeZYW!0@jtO=f6MLp+)lF5Ix^BDYm(9Ymjq zI?%Jf>^;by|FAanbXn{16xKGO*Gs(o5>vFQ^p8Tr_EkXD=>_7e-POk#F9f^woWmSY zwC;G{*^Um3r1IF_J+;}8>A>CZ!T(nE+^^yNq>L;w_eDv-SAU|kXIiM?_K}HE{`LsP zel;aBs`Q`hgezK{=w3C<9s0!+7-x2)-oX~srO&5wb8ajdY@Blzd9*4;WLLR$IHDS}o&ayxi=^SZ4j!AzQ>hjX|gW z=*mR2o|#&qC8sB}Wy)@sVuiracKVJjZ<2Yn{J8u1m)}laNvNgh-4S459;(^zbq+d& zh5764&Hr|}*uwFDyt(-W`Xv-UEuPj6P*8{$*v{Sjm$&}dBg9d=x$vYeGt9EWi|0K! z8*vSv!#m?FW%9>tC2j@JM_2;mkKYWmCZp!!%+FhanRBdl8BlGrR2*lXMqqyCu(x*a zw>{=>EpY<26LB+(lH zu?E_-`(rr$;G092CoZ9A&OXpw@E!@mUkay$|sP9HTRpx@DLU^SAHlAsAG7YWC01 z7N)&Fyf9*x7vG#_5cj9TgVnIe`|KBUh6s>#e5QVrc+sD`c&0f$@->fQr^1xy6>`h& zyCqO;)MhRmrIA!P(QMm=_8Qy%d~>tC#$g1Or@6V?#Y33Tl3_dO*DowrYT5P~blzeT3?KzLU$!Gtn5vsd- z;SPdh?w3$^A)|0!KSzOuYVQX=|GWFhej3-mG3^4e_@_R62!y`|2F)*!tso%te~SOt zyX+5ZV){q;_8?a`h|6y`a{vE;`CkI&&#C#~W|E)M?r3HM0lWRnoqt30AA$H++@Iev z|2Hm(|M%Mc?}tJj-v9Do$j_@uCDdfezcm>O4Tb!Z~4CV%#@qvKmTwnnZ zH$VW)%_j&lGqbSZ`cDx3+lapy09y&8a_T3% zw0Z>LkIn>*Zx>1~{#oIAqwAnA<`e_j*K+&0$WG4rE806SJSlu8Vs|V%>?$@CKU~QI zV<(0V<|IjHXvy7wV5i`vweptS-EyYTH^Z+LS*8P6v>JGpv7d5K&(W2FC3DZ*GQ zbLyZxJ#h7hgFt_I2I%h{#LxY^gEFCx`S51ce51fz1Q}{9NWd z5CES51OVU`0P+c%37A3nfIy%*2nYmN0C;|L(7(U@;r%ZLz{bMa-Fd;*X(~ZGD)#oW z31Ik9UjMxOY*dJLi25T3IRpW!IOPwUFXqmVZ9K^tbM8(?%a8c|cm#WgzPAA#MLmGS zc3DY;eI;v*otdF*jEYPIyOxD_uQpKN`_|19CQhwIQwerU%-YjTIL>HNO_Y&LZAVZH z!2;dJrD5Ta3{K0$g+4P;e?|4#k^}7`aq7V=rWmo9g!i4AU*eawQAw=ht=*eeKNHU` zDJ8_uV!h+fQhx@H&2u2P`5vewdHjWsk4fuO9!n+?a%8DuJ?H%-12-t z*XHYrnDTQ=TD1Ww6rF6$*qkHUbjOhCx@A!Ez38+QEHa;$)KOHEWQ>RND`P}q)2`D; za8Mm3w6PQ_xN5_ig%KRsF^_i&XkA}sszY`pfh%fwZ~o|ZC1zqB{=C~+W}Rst@a^e% zkFAGADPP8Ll18ug0F9NU{1+Bj{Svz1+#>U?4nZXXB(e6(BG{&AU5Q|CEBotx_^@Df zQJpM|+GZIemV~@?s#Js<6rH}Kg;RwA^8GoinF5L_7F z{C!@hj?Lt=g~rIpzzz7ktCy}g&Xny7-UFMVt)RjmDL>HaXk=67XzbtStUnE#_vjfT z-ptUAkR`3EYpob)+?&>Gzj78f+PWgLXQDEYkfAh0>bXMoP8EO`Sa~FZ3o-9hTBIz% z@HBi8sujOex1(P+)f(Nl{liz_`87O}eo*2=4mOnCzE6Ut!b3XzuQ_PJ^FQVw%`bBh z3e5}T<_3Vw0NmVM+=2oC5EmCeh>x2O2oT`q7JwE@3oZyZFW)b7kOpoAnuBecN>MXh z*xRigW^rfr3dK}a_)Z`}DlNEBL^|#F!c~kBMPP&a8Hd1&4jvEOsdxdWoQAh+sh2w? z5%n0FTJC3`KH`6=P_3SqIIk|QD0JL-M4N9~#SGjoF4fWUX(+09Z!2^BP{F@_Y4!$F znPHD&vm=Y%It4dX{7_I@Q%%Ec?bzc>*DW1RJYV=zTV&eHvBB3v_8z72f(*lNhXDb1 zaYz_1k5z4#OnL@X(x&Ghrv@6uC3+pG?WfV{-0?1b-6~a~zmpwX9TuPQEn1grr*`a_ zhIh{ClsS1dN}v+Jzf}bq4t=@qkiYxMy^iDKwuB3j^Z>+=X0`n}Qq`@8x|{M=g$7L} zy9nDg)_Q9-MNZHt{5OPGy&AU*qaU0;;t1f$ieOuXbJIP2T@#j0PJ zA`w}Olm_ezGHRtGbRybaQ%QM{et)1&iVH(XDX!OIGzrpho@@zy)eI(WDuZ}T(wG2* zhFIPuhld|A_An3KZxW5)$u_pobi)pr{oW#gLLR+UYL68E$u&-3yq*aCXEnFs$rsk5 z!j(P~-ASIwH#dFcBHVtgDU95gaW^H;h3^m>=v8F4k~v9@v(}$d)R2;t_2%G(%oHUZ z=kCpB8+`hd7PYa`a9L7ZoM@$jv+$Vv`culzd3f$f8gW`w7)5$Li&^eBI_`VfEm=DQ zQSDU##|J%028)0`Hx(U~4%h-eDe8xmh>w{KVT4BJLv;SHDPhk0Kcmp z2SZ6tti}Q8$(hUGi;*MQ4xkF$%mZx zA6yKPzqqR)6u-GqUVdWL6Av54)BTz_P^_lsYVDXKXkt{qSxTUz`8!IT8iv>Y+Xg&t z-EE~%0^Cjl`pU&@*bzFV+<>^bMC=(wv=dRIr;_%IXgh-d-o1q4>dFy~ zXVgZ`GM|r+B!_Ed^nz3vSXCY5`!7(M5tmK(tw)lW8+k_UHlEn(HMn-lqt&*$wPsrf zso91zkazRy78g*j3J7nVYjL8NaLFA#K92gGg0Bf!tk&&Oj9Gzb1tjN8M+ zK*cyKvx1Tq7cZ|l7cUnV4?vBJ7XX|YVHuwQ%jr8c12#^3LV_iZ-U(j>C9bg~624*P z=HvBay8c{FSEm+g{RU4`TeUGzqbC$A$`B`vjVi4CcA?wV*hKmwC{4O=8-w1VD73km06(u8AHNwd01N^{z|bPj zXTb{wn?rcP0$>P)3&0KG`87M7Fq40A{< zCve@+hL^8ypfan9eNp)Anue}+iQUc=Xd@B*R5J&uS4z@!QQ321F~e0P@whzj{`eW; z6f)(rm$sJeBcq-Lc1n+CdOj`q>g2DL(zbbd@13%cu<1WZ7=c3j3Pb?2Wz37>I;tUh|uPCcHrA475)5< zN4g{=Eq-2p6ajt>3Il@Ws`|6pHpt8H_kz{^Rr{|(-3lVvSAK9C`9^K=f6_YNS~#$E zfi^bGM-c?i8+uJuUuUX>EC^(SCCrkGUM;=1n7;C4 za8iG|?qyDSS0Ey1{m3lFWJ>YiNmRLJSN2jT>1Stej4UsLJuQ6A0f&4ZRuzs|Y;vuF zLtWP|@ui{*O+6{*qXXuY@SiHZni-8HBY8Pl)9~6eikv7E1GpDiQ*j(p!*2H;dRltLig>dz2jhNkdALzBGHU`EuRWWAmi(A&k}_h`r$Cj^D4OWm zlDUiTVwnZO8q7{MgW**0Cy30lg!~>){Eik3rjg#4h$0Gr;Hc z%a_O0gZxns)#RSZRZIdJv=4#%zh(>HzvSWH1nbYU1^$=W0);l?140BrykIVKK@b=S z-~xdGKyHXRABdk90OI8W3YrUYgMOb{{EY#y@Nl=#Y$?-FeAqGV6uxy?)s~NspGR%1 zB#UP8>xIKbIa2;C4?dVrII1K`x^kjb6*boiU>X8r5QvlOB<sb5i(vhz(a{0=JvT<>p|4?s0_W4Zx0;JE>!*I&?(jd40kJi?P=eX_w;94QaG2 zzRau`8kP~b=TdB->SYJMq$5>_Ppah?=exl00_%Kkb(srUne^fjzHKlQ*ZvH+a@G9v z+w{7vSH11KuL386&X7+Aw?7%BPl&+;%5fb&#C#vZ8Y2oc^d7?RkC>nNKXnLx$nPEE z!$0W|e}?{b2ZcgI%y|SsKmk4w^eVR*50IOW$BfUy+??NB00Px+(2+mT!or;UH+}Ip z29%i2e)x8Z!N;Wt;4(M0S#h(DR7$-*sYU;MxO=bxAG>;4J+m-_k}rsU!wE%7~o6GD5G4MCX_lda=quOysav-eUIw zN4Ky8u3axzF2I6=NdAv7;RKqrv*LC4a1E|xBLub!pIzylJf4tnyAt8D?~mjET6a^R z+6pVWYN7k!D3$cx#D`}`A2|D8!x;Flf%E^_QNP}VK%x1-KnpO4hsVO)3<9x$Zt(%U zU;qdJwtxsgCtU&-Kz?pPKJYIc^{Y*Yl{waJ_1d?K*icmI8*oQiob~y|Jd%dFjTM|4 zUUi*A0@g|CCY){Wc{zuxVYjH~v{@hxS_sVu>5vAGq<{ee=?@|K_B3YR6RB@}Fv)LO z1s~YW$ywa`|87FM|8j2o_v3sZ`1c|q^-oT3t^f3g#{{8QkbaQ{U`~DjzXi7duOL)m zgUx|_yyigY9K{U6Yc9ye!^g!1;THh&a|3>3*Z05wxiVNm#QtmOC^69p`cx|38_yLJ z=d9Gar)9eJGjuWMp zpgL*Xl~~@eyE^2wr+9UwgTb`Dq-RLM)O!r4+d>Sp$5Gr+4>PT(LTVZbmUepUu_Yaa=&r&2%0^^}%!Z&cb2=PSCK?PsGxc z^IxEpqs6!q>m-SQWQK`B1mD7w!d_s?XA`6p&liy`$jD5T@`9FBX~F5cA)jkp2Yfjh zW$0qUjXa(&Fw~;+Ie10ozeD67&mezssooT8BQ+6QRK3WyD)ua$^~lN6GRIIqLV|7y z=UkHhXnZ90Y&U&q8C~dXWMpOBLGa{>jqPIq+|3|_Z&-6#w92YlEz(i*wOa4Gz=CLm z<$3!SMGK7XXJSLG5Dl%t7XPy2 zg9uaG-3<>pg*iHVpASxW(g7ZKKSXS}T*N3aE;3%G%C;%jdg7c1`&hka`dsn#8!IB> zj8B5W%5f#K8^-mKv18ynwzxsB?-N%mHFEJX}Usq?S zepE2&(Im$q97bO3FBuCcPd=C6VKCfn>?|wi`_M<6k1Nsp8V@NaNFAg%!WNYCigAWb z%Ndo)&%EkAn2W)aeEH6@DQB{M%o2YjN@4Ml|GAuOS6$O52 ziQ&)7gpemqgrWBmE55hg$eL44=1{cv6J%fr*IY$F2>ghG@D=G!9Zkfj z0NORoEkNAA3#Pv@0JaWB;WxpgWCOk0FY|CR&-sWJA|sX^SMCCvX==g7=s(C2slRu$ znP5m2=Xl=Mx4A#9e{=!!W%{FfX%^D%6a)tK;4xzVqaa?JG!!D-$S~?;72}a*`*$$T zsf3IiadAd?+evYTxOo7Xv8^_u9|wt$x}JPr*BbbV+}j@wOO*3#6ziMIjmDSFwVsDe z;7Ana1)VJTqzH$qwp$IbvinX6K=R21WSM=YHa2sU$9e}-j0@!Q(9aK1ehf83$$B8{ z4=??BcJpg53H(de{>?t~4=?>0+WnUsT_`jckPl$NYi7a22bD{_&`A&%klW0H&)k9^ zBnW`cgm}$(1X zY^(j$TY7y_Qp)x2^g^OXl9NbKr?2YqFaMq^aqc4D<0vb9SkifplOXg%RsNt@cl50` zzKZzuw$<;0%x_lp^DeVURjiyr5_w>A>_-cMbLDfVq-w!Z<5WRC7k-&}3(3|Q2X4vk zAzX^iZParOy%&=_c`Eg3E2*jfYe@EndcLYY?$@rBExZGs7d98^pFj4|Xn|?7O^a|Y z-`{oTdK+|it=VYsBxA~JABxuOgAH?^+7ZtxJXD)pSJYq<=Vz(YaBR2LLka$s_e+no zO4~Oki5OjT)4YD2xFfe$QY<($C1Rb)lr8hc_gi%QZTR{-e`IG|)*j9Gn&CW2 z|NP>vCp`)*>Cey9+7THU9Q>~2v&U&L%lL!W&i6r_9XLF5mccJ9_Os0D_nrjfRm7ZS zaM$cZ_ej@B+?7aDv$eA#B3ni)6*XjEtwBKINCZ0qGSNsX$|Rx+Nn>c$jPU@>tu@_xA#6f7EdeKUEfw#-qp6mL;|tJWWFA~Y~as;bEAgGd%6Ao39l?w7cLN;;8HcpjbMNn6{RES7OLc@OeL=EEs zn8x|zI601`e>d9nz5Dh$u%P8jpM1mpf&lwD)48gBe;N(h=9EM@ZKUI{Dn+rkqIOop zTTxs>V%R4&**Xr=wK7rJnnbWyA`xw^G>~wNI?38vS(!jmMLa;+M%`LNg`lJ~{MCiw zxsxNKy0ko-eQ13OZ%ka3CU@D*+sMrk)jV?> zm}AFf54C{`x8SLnZZu9)S&ze1aMuy+O=Yb}BB&uhO4dYcBFRo&gQ#Y0gGBmlREfws zOx0FR%|=62Ny&QX{ev-9lOi}=z*?+=ELgNagxmC0+@XA@-uGgbCx{IfmmKVIGPo?w zvpz4s#%ofOM}xKf)i}zHZL)ik=gwFX${BxjL3Y9GTj7M%^>x2>#3GZ@xkipWBod z;pV{?=9>`UUAmq0$xI#$!3El!@@T{oz#7aUU`;QO#8i_GWdsb_fNg8%P8IU))lGrN}>CG^(Ycu35z03_Svi?31Uk za0Ul~oBKgv3R%xoF!-k-&2{C8^(Gs;By>#AzvMQt3U;)uSr$KZ2FtVO+qOcv)f@IkIa$GauzV6j`~o4e=Zg{g69xm z9i#-7M7D|}ob*qhL=`pCvR!+BmjvJcd(gor%a~+uT<=y-;LrbNA+*v(r%6MyMZZ$C zIG6LS%Y%&11b*>F)!C+7W=d-qSN7izkBC=kttgkuNgS9jsoT@aJrG@{6kF_dAmG63 zz-jukWe(L{juA<+R+^KjSX%x4tH<^ey{&d6?wp(xR6~CKbg|Ixl02E_$nqc7q_;fL zmy<1O(kNWBZB0eW5)BN^cFGRuxhn8{b_CknZ7v!uOQoimQ?&J73!No|t&hmvoBV!$ z?9n&xi9mZR-C|d1-d`fFqu;eli}y$M%Xy-zg=vWNcJ^B*M0zXP7?3_NRn^Ks zP+s#{_?#7u+l^(#b1G{nFDebL=-?cHs>X{qZe zvADU5#hTZzUeGt^tUUGC)*cPNOWV~&`A z^`zq>fAz2}Mq&azmDb1pJ(K&Xounh~KTx2t@>%~@DSngN%OWk~f;;;QyXm?OgpF>w zzt*MEPK!Dvc-BW=+_^w1=YQ$t5?_wj+{x~~95uZ!Kr%N=?zVQPUgo9Rivm2hhF*Es zivxnvuH2qhuCPJxs8ztg*N#S^RM|e~vTVyF?*;V2o`k#~Jm3}2^}JtfAF{#+Zah4_ zfeu0sa_}8Uei?>?s}fb14!--hgM&{N2hXPzoyXBuZrP4i1jVYE7slsEXkXt>uwoYo5yR@0Fnf)|A1^t6%>4q+{I^p_j#k4p|Y09;xl-b2G10jp6 zr4e20z@}4zh_3ZgO+>9z@agrD3k;-s&m6avE=!NUJH_Z`q!RZk#o7mH>k|$I9hS)Y z5i8&$O_Y`_*ierK`(7h%B@kOb8-6oa-Dcaar-$FSAwmYHW|cs$$DC zpRGN%YXW*TTzIDPDt$PrCr50edc}n-P1U`Y^LD9n@VqOY?-~?SQWB~X7G^BTGwK() z@3rr%!-Li;r__a{b>R#9w#=xHl1xx8E!T9WJ#>%yocd`8aJh9Y@`}8O%K^t?y4)(r z<*4jjPMzs;cE4Q?e6qM4@=i6W*AdF^mD9T`BHf`*O%}>NE2Mit^u;Dq*;!Jv>ziyh z`g-1zG^RWZ?7X49z--0W=W%grZVgZJ;=OA*e+sNw8B^q_q*?3o%i(C{29C*1wsC99 z0x_?w_Z7hmU zZTgG9H?Z7H_S@pOeVn0*2~Tnpl~c=}|8^ zle@#rxup^XGzHqJC+EfHuW{ci`{tag)~Q>sc@b5@MV)5aN0G3 zHGXRmrvqoV!E$i-anGKYnALj{t38I4Z% zLu#6iWHgSucbMY?fZ!``!1XYUmPM@@6$sy5Jgg>zucrag3`WbMh9DDYB%N_wp2L6w zU9wsZCPXKzAZwbDP{P+vWv*N;K z#L)v?B!q#BRAgcfHqIiptk7@;WAs4l&BTTtSJiS>Sh#pED38o&bgG^xHtc^Wmdpwc z*GUESfff zq(pxuz~w+dH55k6qK1zRg0nUXObCuk0}M2wWaOA-Bq5yTMwo=a(hx3?0k1)_sPwTx z{w#JdBbviN;OSY}fXE@E(a8?T;m8ZCyqmv8SAVINO9C(a{DQrSe zkAu;&s8Yz07#6c(JeB}1`0t}eh?(|eUDJbd6pThE$0&`4%E*lw+B@JJH&z^YRYxax zVdCH_R)O|KxC8?@>|``)s-ZF#r3ouu0ecc$o&h`vn)_)aMQP;7GHi^i4g`}sfPz!s z(cC-OK}}3u2>jec(u6$S~cK`(^ilVvKvV)qK{8;?l>ycCGvFAIkY*jROI1LodoutDC zYGSfi@pJ!<`9QdGQPJGt6i+mFGcp$U3Cm4|xx+fYk?(nNB=$D;xZ}z*WpW2laKav% z`#yG16O(s}pS!*;7I$1(gJ|w>;uf0wC3a8~lSK$~A5EJC#ve2~dHGr_?zpnYnA`yr zoR)>=9>or7VzSBbbKhcs#T{2}7@9kr>4fH<#SUs>a>ej-=QPCRjwjs;%^l7|LUaEg zJE)0Czk<1smi{#C@xbSI8ewwBlMKb=4$`4uWj&hvJ$6tNlN<$ehedIKGkQHALc!#Y zN8Zom4xnJYI-2`Oc2E-|_lLP-f5#Ebu(;zA%`>?JC|E^|<{rupYGOq6Fn8>6kFvnx zj!VkS zm=AT8=IkSP-7gkBa z6Df-tJ0=Eh=`Ul}0$%{aOIRO=2J^ESix3g73=;+ek~>(}ga+%h!vMo4z%w=EK`!Iv R;MjotEM3gOaoG|4?f>_U4KM%z diff --git a/vulnerabilities/tests/test_data/npm/npm-expected_1.json b/vulnerabilities/tests/test_data/npm/npm-expected_1.json new file mode 100644 index 000000000..b04dc0efe --- /dev/null +++ b/vulnerabilities/tests/test_data/npm/npm-expected_1.json @@ -0,0 +1,62 @@ +{ + "aliases": [], + "summary": "Versions of `@hapi/subtext` prior to 6.1.3 or 7.0.3 are vulnerable to Denial of Service. The Content-Encoding HTTP header parser has a vulnerability which will cause the function to throw a system error if the header contains some invalid values. Because hapi rethrows system errors (as opposed to catching expected application errors), the error is thrown all the way up the stack. If no unhandled exception handler is available, the application will exist, allowing an attacker to shut down services.", + "affected_packages": [ + { + "package": { + "type": "npm", + "namespace": "@hapi", + "name": "subtext", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:npm/>=4.1.0|<6.1.3|>=7.0.0|<7.0.3", + "fixed_version": "6.1.3" + }, + { + "package": { + "type": "npm", + "namespace": "@hapi", + "name": "subtext", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:npm/>=4.1.0|<6.1.3|>=7.0.0|<7.0.3", + "fixed_version": "7.0.3" + }, + { + "package": { + "type": "npm", + "namespace": "@hapi", + "name": "subtext", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:npm/>=4.1.0|<6.1.3|>=7.0.0|<7.0.3", + "fixed_version": "7.0.4" + }, + { + "package": { + "type": "npm", + "namespace": "@hapi", + "name": "subtext", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:npm/>=4.1.0|<6.1.3|>=7.0.0|<7.0.3", + "fixed_version": "8.0.0" + } + ], + "references": [ + { + "reference_id": 1476, + "url": "https://registry.npmjs.org/-/npm/v1/advisories/1476", + "severities": [] + } + ], + "date_published": "2020-02-18T18:00:29.843000+00:00" +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/npm/npm-expected_2.json b/vulnerabilities/tests/test_data/npm/npm-expected_2.json new file mode 100644 index 000000000..ed5742922 --- /dev/null +++ b/vulnerabilities/tests/test_data/npm/npm-expected_2.json @@ -0,0 +1,64 @@ +{ + "aliases": [ + "CVE-2020-11022" + ], + "summary": "Versions of `jquery` prior to 3.5.0 are vulnerable to Cross-Site Scripting. Passing HTML from untrusted sources - even after sanitizing it - to one of jQuery's DOM manipulation methods (i.e. .html(), .append(), and others) may execute arbitrary JavaScript in a victim's browser.", + "affected_packages": [ + { + "package": { + "type": "npm", + "namespace": null, + "name": "jquery", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:npm/<3.5.0", + "fixed_version": "3.5.0" + }, + { + "package": { + "type": "npm", + "namespace": null, + "name": "jquery", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:npm/<3.5.0", + "fixed_version": "3.5.1" + }, + { + "package": { + "type": "npm", + "namespace": null, + "name": "jquery", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:npm/<3.5.0", + "fixed_version": "3.6.0" + }, + { + "package": { + "type": "npm", + "namespace": null, + "name": "jquery", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:npm/<3.5.0", + "fixed_version": "3.6.1" + } + ], + "references": [ + { + "reference_id": 1518, + "url": "https://registry.npmjs.org/-/npm/v1/advisories/1518", + "severities": [] + } + ], + "date_published": "2020-04-30T18:19:09.542000+00:00" +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/npm/npm-expected_3.json b/vulnerabilities/tests/test_data/npm/npm-expected_3.json new file mode 100644 index 000000000..00cfdaa3c --- /dev/null +++ b/vulnerabilities/tests/test_data/npm/npm-expected_3.json @@ -0,0 +1,158 @@ +{ + "aliases": [], + "summary": "Version of `kerberos` prior to 1.0.0 are vulnerable to DLL Injection. The package loads DLLs without specifying a full path. This may allow attackers to create a file with the same name in a folder that precedes the intended file in the DLL path search. Doing so would allow attackers to execute arbitrary code in the machine.", + "affected_packages": [ + { + "package": { + "type": "npm", + "namespace": null, + "name": "kerberos", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:npm/<1.0.0", + "fixed_version": "1.0.0" + }, + { + "package": { + "type": "npm", + "namespace": null, + "name": "kerberos", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:npm/<1.0.0", + "fixed_version": "1.1.0" + }, + { + "package": { + "type": "npm", + "namespace": null, + "name": "kerberos", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:npm/<1.0.0", + "fixed_version": "1.1.1" + }, + { + "package": { + "type": "npm", + "namespace": null, + "name": "kerberos", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:npm/<1.0.0", + "fixed_version": "1.1.2" + }, + { + "package": { + "type": "npm", + "namespace": null, + "name": "kerberos", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:npm/<1.0.0", + "fixed_version": "1.1.3" + }, + { + "package": { + "type": "npm", + "namespace": null, + "name": "kerberos", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:npm/<1.0.0", + "fixed_version": "1.1.4" + }, + { + "package": { + "type": "npm", + "namespace": null, + "name": "kerberos", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:npm/<1.0.0", + "fixed_version": "1.1.5" + }, + { + "package": { + "type": "npm", + "namespace": null, + "name": "kerberos", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:npm/<1.0.0", + "fixed_version": "1.1.6" + }, + { + "package": { + "type": "npm", + "namespace": null, + "name": "kerberos", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:npm/<1.0.0", + "fixed_version": "1.1.7" + }, + { + "package": { + "type": "npm", + "namespace": null, + "name": "kerberos", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:npm/<1.0.0", + "fixed_version": "2.0.0-beta.0" + }, + { + "package": { + "type": "npm", + "namespace": null, + "name": "kerberos", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:npm/<1.0.0", + "fixed_version": "2.0.0" + }, + { + "package": { + "type": "npm", + "namespace": null, + "name": "kerberos", + "version": null, + "qualifiers": null, + "subpath": null + }, + "affected_version_range": "vers:npm/<1.0.0", + "fixed_version": "2.0.1" + } + ], + "references": [ + { + "reference_id": 1514, + "url": "https://registry.npmjs.org/-/npm/v1/advisories/1514", + "severities": [] + } + ], + "date_published": "2020-04-14T21:44:49.820000+00:00" +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/npm/npm_test_1.json b/vulnerabilities/tests/test_data/npm/npm_test_1.json new file mode 100644 index 000000000..62a0cb816 --- /dev/null +++ b/vulnerabilities/tests/test_data/npm/npm_test_1.json @@ -0,0 +1,32 @@ +{ + "id": 1476, + "created": "2020-02-17T13:39:01.39", + "updated_at": "2020-02-18T18:00:29.843", + "deleted": null, + "title": "Denial of Service", + "found_by": { + "link": "", + "name": "Eran Hammer", + "email": "" + }, + "reported_by": { + "link": "", + "name": "Eran Hammer", + "email": "" + }, + "module_name": "@hapi/subtext", + "cves": [], + "vulnerable_versions": ">=4.1.0 <6.1.3 || >= 7.0.0 <7.0.3", + "patched_versions": ">=6.1.3 <7.0.0 || >=7.0.3", + "overview": "Versions of `@hapi/subtext` prior to 6.1.3 or 7.0.3 are vulnerable to Denial of Service. The Content-Encoding HTTP header parser has a vulnerability which will cause the function to throw a system error if the header contains some invalid values. Because hapi rethrows system errors (as opposed to catching expected application errors), the error is thrown all the way up the stack. If no unhandled exception handler is available, the application will exist, allowing an attacker to shut down services.", + "recommendation": "Upgrade to version 6.1.3 or 7.0.3", + "references": "", + "access": "public", + "severity": "high", + "cwe": "CWE-400", + "metadata": { + "module_type": "", + "exploitability": 6, + "affected_components": "" + } +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/npm/npm_test_2.json b/vulnerabilities/tests/test_data/npm/npm_test_2.json new file mode 100644 index 000000000..d4fcb40cc --- /dev/null +++ b/vulnerabilities/tests/test_data/npm/npm_test_2.json @@ -0,0 +1,34 @@ +{ + "id": 1518, + "created": "2020-04-30T18:19:09.542Z", + "updated_at": "2020-04-30T18:19:09.542Z", + "deleted": null, + "title": "Cross-Site Scripting", + "found_by": { + "link": "", + "name": "Masato Kinugawa", + "email": "" + }, + "reported_by": { + "link": "", + "name": "Masato Kinugawa", + "email": "" + }, + "module_name": "jquery", + "cves": [ + "CVE-2020-11022" + ], + "vulnerable_versions": "<3.5.0", + "patched_versions": ">=3.5.0", + "overview": "Versions of `jquery` prior to 3.5.0 are vulnerable to Cross-Site Scripting. Passing HTML from untrusted sources - even after sanitizing it - to one of jQuery's DOM manipulation methods (i.e. .html(), .append(), and others) may execute arbitrary JavaScript in a victim's browser.", + "recommendation": "Upgrade to version 3.5.0 or later.", + "references": "- [GitHub Advisory](https://github.com/advisories/GHSA-gxr4-xjj5-5px2)", + "access": "public", + "severity": "moderate", + "cwe": "CWE-79", + "metadata": { + "module_type": "", + "exploitability": 3, + "affected_components": "" + } +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/npm/npm_test_3.json b/vulnerabilities/tests/test_data/npm/npm_test_3.json new file mode 100644 index 000000000..f347221ab --- /dev/null +++ b/vulnerabilities/tests/test_data/npm/npm_test_3.json @@ -0,0 +1,32 @@ +{ + "id": 1514, + "created": "2020-04-14T21:44:49.820Z", + "updated_at": "2020-04-14T21:44:49.820Z", + "deleted": null, + "title": "DLL Injection", + "found_by": { + "link": "", + "name": "Dan Shallom, OP Innovate Ltd", + "email": "" + }, + "reported_by": { + "link": "", + "name": "Dan Shallom, OP Innovate Ltd", + "email": "" + }, + "module_name": "kerberos", + "cves": [], + "vulnerable_versions": "<1.0.0", + "patched_versions": ">=1.0.0", + "overview": "Version of `kerberos` prior to 1.0.0 are vulnerable to DLL Injection. The package loads DLLs without specifying a full path. This may allow attackers to create a file with the same name in a folder that precedes the intended file in the DLL path search. Doing so would allow attackers to execute arbitrary code in the machine.", + "recommendation": "Upgrade to version 1.0.0 or later.", + "references": "", + "access": "public", + "severity": "high", + "cwe": "CWE-114", + "metadata": { + "module_type": "", + "exploitability": 4, + "affected_components": "" + } +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_npm.py b/vulnerabilities/tests/test_npm.py index eccbaaa04..d885679a7 100644 --- a/vulnerabilities/tests/test_npm.py +++ b/vulnerabilities/tests/test_npm.py @@ -7,159 +7,39 @@ # See https://github.com/nexB/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # - +import json import os -import shutil -import tempfile -import zipfile -from unittest.mock import patch from django.test import TestCase -from univers.versions import SemverVersion -from vulnerabilities import models -from vulnerabilities.import_runner import ImportRunner -from vulnerabilities.importers.npm import categorize_versions -from vulnerabilities.importers.npm import normalize_ranges -from vulnerabilities.package_managers import NpmVersionAPI -from vulnerabilities.package_managers import PackageVersion +from vulnerabilities.importers.npm import parse_advisory_data +from vulnerabilities.tests import util_tests BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/") - +TEST_DATA = os.path.join(BASE_DIR, "test_data/npm") -# MOCK_VERSION_API = NpmVersionAPI( -# cache={ -# "jquery": {Version("3.4.0"), Version("3.8.0")}, -# "kerberos": {Version("0.5.8"), Version("1.2.0")}, -# "@hapi/subtext": { -# Version("3.7.0"), -# Version("4.1.1"), -# Version("6.1.3"), -# Version("7.0.0"), -# Version("7.0.5"), -# }, -# } -# ) -# -# @patch ( "vulnerabilities.importers.NpmImporter._update_from_remote" ) class NpmImportTest(TestCase): - tempdir = None - # - # @classmethod - # def setUpClass( cls ) -> None : - # cls.tempdir = tempfile.mkdtemp () - # zip_path = os.path.join ( TEST_DATA, "npm.zip" ) - # - # with zipfile.ZipFile ( zip_path, "r" ) as zip_ref : - # zip_ref.extractall ( cls.tempdir ) - # - # cls.importer = models.Importer.objects.create ( - # name="npm_unittests", - # license="", - # last_run=None, - # data_source="NpmImporter", - # data_source_cfg={ - # "repository_url": "https://example.git", - # "working_directory": os.path.join(cls.tempdir, "npm/npm_test"), - # "create_working_directory": False, - # "remove_working_directory": False, - # }, - # ) - - # @classmethod - # def tearDownClass( cls ) -> None : - # # Make sure no requests for unexpected package names have been made during the tests. - # shutil.rmtree ( cls.tempdir ) - # assert len ( MOCK_VERSION_API.cache ) == 3, MOCK_VERSION_API.cache - # - # def test_import( self, _ ) : - # runner = ImportRunner ( self.importer, 5 ) - # - # with patch ( "vulnerabilities.importers.NpmImporter.versions", new=MOCK_VERSION_API ) : - # with patch ( "vulnerabilities.importers.NpmImporter.set_api" ) : - # runner.run () - # - # assert models.Vulnerability.objects.count () == 3 - # assert models.VulnerabilityReference.objects.count () == 3 - # assert models.PackageRelatedVulnerability.objects.all ().count () == 4 - # - # assert models.Package.objects.count () == 8 - # - # self.assert_for_package ( - # "jquery", {"3.4.0"}, {"3.8.0"}, "1518", vulnerability_id="CVE-2020-11022" - # ) # nopep8 - # self.assert_for_package ( "kerberos", {"0.5.8"}, {"1.2.0"}, "1514" ) - # self.assert_for_package ( "subtext", {"4.1.1", "7.0.0"}, {"6.1.3", "7.0.5"}, "1476" ) - # - # def assert_for_package( - # self, - # package_name, - # impacted_versions, - # resolved_versions, - # vuln_id, - # vulnerability_id=None, - # ) : - # vuln = None - # - # for version in impacted_versions : - # pkg = models.Package.objects.get ( name=package_name, version=version ) - # - # assert pkg.vulnerabilities.count () == 1 - # vuln = pkg.vulnerabilities.first () - # if vulnerability_id : - # assert vuln.vulnerability_id == vulnerability_id - # - # ref_url = f"https://registry.npmjs.org/-/npm/v1/advisories/{vuln_id}" - # assert models.VulnerabilityReference.objects.get ( url=ref_url, vulnerability=vuln ) - # - # for version in resolved_versions : - # pkg = models.Package.objects.get ( name=package_name, version=version ) - # assert models.PackageRelatedVulnerability.objects.filter ( - # patched_package=pkg, vulnerability=vuln - # ) - - -def test_categorize_versions_simple_ranges(): - all_versions = {PackageVersion("3.4.0"), PackageVersion("3.8.0")} - impacted_ranges = "<3.5.0" - resolved_ranges = ">=3.5.0" - - impacted_versions, resolved_versions = categorize_versions( - all_versions, impacted_ranges, resolved_ranges - ) - - assert impacted_versions == {SemverVersion("3.4.0")} - assert resolved_versions == {SemverVersion("3.8.0")} - - -def test_categorize_versions_complex_ranges(): - all_versions = { - PackageVersion("3.7.0"), - PackageVersion("4.1.1"), - PackageVersion("6.1.3"), - PackageVersion("7.0.0"), - PackageVersion("7.0.5"), - } - impacted_ranges = ">=4.1.0 <6.1.3 || >= 7.0.0 <7.0.3" - resolved_ranges = ">=6.1.3 <7.0.0 || >=7.0.3" - - impacted_versions, resolved_versions = categorize_versions( - all_versions, impacted_ranges, resolved_ranges - ) - - assert impacted_versions == {SemverVersion("4.1.1"), SemverVersion("7.0.0")} - assert resolved_versions == { - SemverVersion("3.7.0"), - SemverVersion("6.1.3"), - SemverVersion("7.0.5"), - } - - -def test_normalize_ranges(): - assert normalize_ranges(">=6.1.3 < 7.0.0 || >=7.0.3") == [">=6.1.3,<7.0.0", ">=7.0.3"] - assert normalize_ranges(">=4.1.0 <6.1.3 || >= 7.0.0 <7.0.3") == [ - ">=4.1.0,<6.1.3", - ">=7.0.0,<7.0.3", - ] + def test_npm_advisories_1(self): + with open(os.path.join(TEST_DATA, "npm_test_1.json")) as f: + mock_response = json.load(f) + expected_file = os.path.join(TEST_DATA, f"npm-expected_1.json") + imported_data = parse_advisory_data(mock_response) + result = imported_data.to_dict() + util_tests.check_results_against_json(result, expected_file) + + def test_npm_advisories__2(self): + with open(os.path.join(TEST_DATA, "npm_test_2.json")) as f: + mock_response = json.load(f) + expected_file = os.path.join(TEST_DATA, "npm-expected_2.json") + imported_data = parse_advisory_data(mock_response) + result = imported_data.to_dict() + util_tests.check_results_against_json(result, expected_file) + + def test_npm_advisories__3(self): + with open(os.path.join(TEST_DATA, "npm_test_3.json")) as f: + mock_response = json.load(f) + expected_file = os.path.join(TEST_DATA, "npm-expected_3.json") + imported_data = parse_advisory_data(mock_response) + result = imported_data.to_dict() + util_tests.check_results_against_json(result, expected_file) diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index 6b3ae0808..737b26171 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -29,7 +29,6 @@ from packageurl import PackageURL from univers.version_range import RANGE_CLASS_BY_SCHEMES from univers.version_range import VersionRange -from univers.versions import Version logger = logging.getLogger(__name__) @@ -40,22 +39,8 @@ @dataclasses.dataclass(order=True, frozen=True) class AffectedPackage: - package: PackageURL - affected_version_range: Optional[VersionRange] = None - fixed_version: Optional[Version] = None - - def to_dict(self): - """ - Return a serializable dict that can be converted back using self.from_dict - """ - affected_version_range = None - if self.affected_version_range: - affected_version_range = str(self.affected_version_range) - return { - "package": self.package.to_dict(), - "affected_version_range": affected_version_range, - "fixed_version": str(self.fixed_version) if self.fixed_version else None, - } + vulnerable_package: PackageURL + patched_package: Optional[PackageURL] = None def load_yaml(path): @@ -193,8 +178,8 @@ def nearest_patched_package( affected_package_with_patched_package_objects.append( AffectedPackage( - package=vulnerable_package.purl, - fixed_version=patched_package.purl if patched_package else None, + vulnerable_package=vulnerable_package.purl, + patched_package=patched_package.purl if patched_package else None, ) )