diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 01418af19..d05b5f0ba 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -38,7 +38,7 @@ jobs: pip install -r requirements.txt - name: Run tests - run: python -m pytest + run: python -m pytest -v -m "not webtest" env: # The hostname, username used to communicate with the PostgreSQL service container POSTGRES_HOST: localhost diff --git a/.github/workflows/upstream_test.yml b/.github/workflows/upstream_test.yml new file mode 100644 index 000000000..30cbcdd90 --- /dev/null +++ b/.github/workflows/upstream_test.yml @@ -0,0 +1,51 @@ +on: + workflow_dispatch: # allow manual execution + push: + schedule: + # run on every 9 o'clock + - cron: '0 9 * * *' + +jobs: + unit_tests: + runs-on: ubuntu-latest + + services: + # Label used to access the service container + postgres: + image: postgres + env: + POSTGRES_PASSWORD: vulnerablecode + POSTGRES_DB: vulnerablecode + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # Maps tcp port 5432 on service container to the host + - 5432:5432 + steps: + - name: Check out repository code + uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Install dependencies + run: | + sudo apt install python3-dev postgresql libpq-dev build-essential libxml2-dev libxslt1-dev + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run tests + run: pytest -v -m webtest + env: + # The hostname, username used to communicate with the PostgreSQL service container + POSTGRES_HOST: localhost + VC_DB_USER: postgres + POSTGRES_PORT: 5432 + DJANGO_DEV: 1 + GH_TOKEN: 1 \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index e0f5a631a..e47738ee8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ before_script: script: - ./manage.py collectstatic - - python -m pytest + - python -m pytest -v -m "not webtest" notifications: email: false diff --git a/AUTHORS.rst b/AUTHORS.rst index 039459f93..baf9bc569 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -11,4 +11,5 @@ The following organizations or individuals have contributed to this repo: - Islam Elhakmi @EslamHiko - Edoardo Lanzini @elanzini - Navonil Das @NavonilDas -- Tushar Upadhyay @tushar912 \ No newline at end of file +- Tushar Upadhyay @tushar912 +- Hritik Vijay @hritik14 diff --git a/Dockerfile b/Dockerfile index 3cc935436..c4308986e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,8 @@ ENV PYTHONUNBUFFERED 1 RUN mkdir /vulnerablecode WORKDIR /vulnerablecode ADD . /vulnerablecode/ -RUN pip install -r requirements.txt +RUN pip install -r requirements.txt && \ + DJANGO_DEV=1 python manage.py collectstatic LABEL "base_image": "pkg:docker/python@sha256%3Ae9b7e3b4e9569808066c5901b8a9ad315a9f14ae8d3949ece22ae339fff2cad0" LABEL "dockerfile_url": "https://github.com/nexB/vulnerablecode/blob/develop/Dockerfile" diff --git a/README.rst b/README.rst index d328634c9..5fa05ceb3 100644 --- a/README.rst +++ b/README.rst @@ -141,11 +141,12 @@ On Debian-based distros, these can be installed with:: **Application dependencies** -Create a virtualenv, install dependencies, and run the database migrations:: +Create a virtualenv, install dependencies, generate static files and run the database migrations:: python3 -m venv venv source venv/bin/activate pip install -r requirements.txt + DJANGO_DEV=1 python manage.py collectstatic DJANGO_DEV=1 python manage.py migrate The environment variable ``DJANGO_DEV`` is used to load settings suitable for @@ -160,6 +161,11 @@ for this purpose:: SECRET_KEY=$(python -c "from django.core.management import utils; print(utils.get_random_secret_key())") +You will also need to setup the VC_ALLOWED_HOSTS environment variable to match the hostname where the app is deployed:: + + VC_ALLOWED_HOSTS=vulnerablecode.your.domain.example.com + +You can specify several host by separating them with a colon `:` Using Nix ~~~~~~~~~ diff --git a/etc/nix/flake.nix b/etc/nix/flake.nix index 10cc4c7cd..ca29e3bc7 100644 --- a/etc/nix/flake.nix +++ b/etc/nix/flake.nix @@ -53,9 +53,9 @@ # mach-nix release) is usually insufficient. Use # ./get-latest-pypi-deps-db.sh to obtain the data rev & hash. pypiDataRev = - "499750266bb4b2840cbe856c2cc0e3297685e362"; # 2021-03-06T08:13:08Z + "e9b0fc6b92cd6efbca7ba3b3d4a551bcc13a73c5"; # 2021-03-27T08:13:04Z pypiDataSha256 = - "188g24k8pk4lgqybywimkvwjwh8014v6l2mrkvzv309882i9p5gc"; + "1ssa48l2iz8kncby1gfrbds79mg114dkhpxrridwcq6q2c37p62s"; }); in { diff --git a/requirements.txt b/requirements.txt index 37d2014cf..6d45f57bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,9 +9,9 @@ contextlib2==0.5.5 decorator==4.4.2 dephell-specifier==0.2.1 dj-database-url==0.4.2 -Django==3.0.7 +Django==3.0.13 django-filter==2.2.0 -djangorestframework==3.11.0 +djangorestframework==3.11.2 django-widget-tweaks==1.4.8 drf-spectacular==0.13.0 gunicorn==19.7.1 @@ -19,7 +19,7 @@ importlib-metadata==1.3.0 ipython==7.13.0 ipython-genutils==0.2.0 jedi==0.17.0 -lxml==4.6.2 +lxml==4.6.3 more-itertools==8.0.2 packageurl-python==0.9.3 packaging==19.2 @@ -34,7 +34,7 @@ ptyprocess==0.6.0 py==1.8.0 pycparser==2.20 pygit2==1.5.0 -Pygments==2.6.1 +Pygments==2.7.4 pyparsing==2.4.5 pytest==5.3.2 django-widget-tweaks==1.4.8 @@ -54,5 +54,5 @@ whitenoise==5.0.1 zipp==0.6.0 requests==2.23.0 toml==0.10.2 -PyYAML==5.3.1 +PyYAML==5.4 freezegun==1.1.0 \ No newline at end of file diff --git a/vulnerabilities/api.py b/vulnerabilities/api.py index 07efcfc55..cb3fc1756 100644 --- a/vulnerabilities/api.py +++ b/vulnerabilities/api.py @@ -152,6 +152,7 @@ def bulk_search(self, request): ) for purl in request.data["purls"]: try: + purl_string = purl purl = PackageURL.from_string(purl).to_dict() except ValueError as ve: return Response(status=400, data={"Error": f"Invalid Package URL: {purl}"}) @@ -165,6 +166,7 @@ def bulk_search(self, request): purl_response = purl purl_response["unresolved_vulnerabilities"] = [] purl_response["resolved_vulnerabilities"] = [] + purl_response["purl"] = purl_string response.append(purl_response) return Response(response) diff --git a/vulnerabilities/data_source.py b/vulnerabilities/data_source.py index 28895a9cb..c911518ec 100644 --- a/vulnerabilities/data_source.py +++ b/vulnerabilities/data_source.py @@ -44,6 +44,7 @@ from vulnerabilities.oval_parser import OvalParser from vulnerabilities.severity_systems import ScoringSystem +from vulnerabilities.helpers import is_cve logger = logging.getLogger(__name__) @@ -88,6 +89,10 @@ class Advisory: resolved_package_urls: Iterable[PackageURL] = dataclasses.field(default_factory=list) references: List[Reference] = dataclasses.field(default_factory=list) + def __post_init__(self): + if self.vulnerability_id and not is_cve(self.vulnerability_id): + raise ValueError("CVE expected, found: {}".format(self.vulnerability_id)) + def normalized(self): impacted_package_urls = {package_url for package_url in self.impacted_package_urls} resolved_package_urls = {package_url for package_url in self.resolved_package_urls} diff --git a/vulnerabilities/helpers.py b/vulnerabilities/helpers.py index 36bf7c865..77376ee94 100644 --- a/vulnerabilities/helpers.py +++ b/vulnerabilities/helpers.py @@ -25,6 +25,7 @@ import requests import toml +import urllib3 import yaml # TODO add logging here @@ -79,3 +80,21 @@ def create_etag(data_src, url, etag_key): is_cve = re.compile(r"CVE-\d+-\d+", re.IGNORECASE).match + + +def requests_with_5xx_retry(max_retries=5, backoff_factor=0.5): + """ + Returns a requests sessions which retries on 5xx errors with + a backoff_factor + """ + retries = urllib3.util.Retry( + total=max_retries, + backoff_factor=backoff_factor, + raise_on_status=True, + status_forcelist=range(500, 600, 1), + ) + adapter = requests.adapters.HTTPAdapter(max_retries=retries) + session = requests.Session() + session.mount("https://", adapter) + session.mount("http://", adapter) + return session diff --git a/vulnerabilities/importers/alpine_linux.py b/vulnerabilities/importers/alpine_linux.py index d3831f47c..cd222ec46 100644 --- a/vulnerabilities/importers/alpine_linux.py +++ b/vulnerabilities/importers/alpine_linux.py @@ -38,6 +38,7 @@ from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource from vulnerabilities.data_source import Reference +from vulnerabilities.helpers import is_cve BASE_URL = "https://secdb.alpinelinux.org/" @@ -193,7 +194,7 @@ def _load_advisories( impacted_package_urls=[], resolved_package_urls=resolved_purls, references=references, - vulnerability_id=vuln_ids[0] if vuln_ids[0] != "CVE-????-?????" else "", + vulnerability_id=vuln_ids[0] if is_cve(vuln_ids[0]) else "", ) ) diff --git a/vulnerabilities/importers/debian.py b/vulnerabilities/importers/debian.py index ae9004f37..28d499980 100644 --- a/vulnerabilities/importers/debian.py +++ b/vulnerabilities/importers/debian.py @@ -45,6 +45,7 @@ def validate_schema(advisory_dict): deb_versions = [ "bullseye", + "bullseye-security", "buster", "buster-security", "sid", diff --git a/vulnerabilities/importers/istio.py b/vulnerabilities/importers/istio.py index fa0f1f903..5e29e739b 100644 --- a/vulnerabilities/importers/istio.py +++ b/vulnerabilities/importers/istio.py @@ -31,6 +31,8 @@ from vulnerabilities.data_source import Advisory, GitDataSource, Reference from vulnerabilities.package_managers import GitHubTagsAPI +is_release = re.compile(r"^[\d.]+$", re.IGNORECASE).match + class IstioDataSource(GitDataSource): def __enter__(self): @@ -47,7 +49,7 @@ def set_api(self): asyncio.run(self.version_api.load_api(["istio/istio"])) def updated_advisories(self) -> Set[Advisory]: - files = self._updated_files + files = self._added_files.union(self._updated_files) advisories = [] for f in files: processed_data = self.process_file(f) @@ -195,5 +197,3 @@ def get_data_from_md(self, path): with open(path) as f: yaml_lines = self.get_yaml_lines(f) return self.get_data_from_yaml_lines(yaml_lines) - - is_release = re.compile(r"^[\d.]+$", re.IGNORECASE).match diff --git a/vulnerabilities/importers/project_kb_msr2019.py b/vulnerabilities/importers/project_kb_msr2019.py index 156cfb82f..b9f797c48 100644 --- a/vulnerabilities/importers/project_kb_msr2019.py +++ b/vulnerabilities/importers/project_kb_msr2019.py @@ -47,7 +47,7 @@ class ProjectKBMSRDataSource(DataSource): CONFIG_CLASS = ProjectKBDataSourceConfiguration - url = "https://raw.githubusercontent.com/SAP/project-kb/master/MSR2019/dataset/vulas_db_msr2019_release.csv" # nopep8 + url = "https://raw.githubusercontent.com/SAP/project-kb/master/MSR2019/dataset/vulas_db_msr2019_release.csv" def updated_advisories(self): if create_etag(data_src=self, url=self.url, etag_key="ETag"): @@ -82,7 +82,7 @@ def to_advisories(csv_reader): summary="", impacted_package_urls=[], references=[reference], - cve_id=vuln_id, + vulnerability_id=vuln_id, ) ) diff --git a/vulnerabilities/importers/redhat.py b/vulnerabilities/importers/redhat.py index 1fba1c484..9415cd94a 100644 --- a/vulnerabilities/importers/redhat.py +++ b/vulnerabilities/importers/redhat.py @@ -20,14 +20,15 @@ # VulnerableCode is a free software code from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. -from packageurl import PackageURL import requests +from packageurl import PackageURL from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource from vulnerabilities.data_source import DataSourceConfiguration from vulnerabilities.data_source import Reference from vulnerabilities.data_source import VulnerabilitySeverity +from vulnerabilities.helpers import requests_with_5xx_retry from vulnerabilities.severity_systems import scoring_systems @@ -43,6 +44,9 @@ def updated_advisories(self): return self.batch_advisories(processed_advisories) +requests_session = requests_with_5xx_retry(max_retries=5, backoff_factor=1) + + def fetch(): """ Return a list of CVE data mappings fetched from the RedHat API. @@ -58,7 +62,7 @@ def fetch(): current_url = url_template.format(page_no) try: print(f"Fetching: {current_url}") - response = requests.get(current_url) + response = requests_session.get(current_url) if response.status_code != requests.codes.ok: # TODO: log me print(f"Failed to fetch results from {current_url}") @@ -90,38 +94,51 @@ def to_advisory(advisory_data): bugzilla = advisory_data.get("bugzilla") if bugzilla: url = "https://bugzilla.redhat.com/show_bug.cgi?id={}".format(bugzilla) - bugzilla_data = requests.get(f"https://bugzilla.redhat.com/rest/bug/{bugzilla}").json() - bugzilla_severity_val = bugzilla_data["bugs"][0]["severity"] - bugzilla_severity = VulnerabilitySeverity( - system=scoring_systems["rhbs"], - value=bugzilla_severity_val, - ) + bugzilla_data = requests_session.get( + f"https://bugzilla.redhat.com/rest/bug/{bugzilla}" + ).json() + if ( + bugzilla_data.get("bugs") + and len(bugzilla_data["bugs"]) + and bugzilla_data["bugs"][0].get("severity") + ): + bugzilla_severity_val = bugzilla_data["bugs"][0]["severity"] + bugzilla_severity = VulnerabilitySeverity( + system=scoring_systems["rhbs"], + value=bugzilla_severity_val, + ) - references.append( - Reference( - severities=[bugzilla_severity], - url=url, - reference_id=bugzilla, + references.append( + Reference( + severities=[bugzilla_severity], + url=url, + reference_id=bugzilla, + ) ) - ) for rh_adv in advisory_data["advisories"]: # RH provides 3 types of advisories RHSA, RHBA, RHEA. Only RHSA's contain severity score. # See https://access.redhat.com/articles/2130961 for more details. if "RHSA" in rh_adv.upper(): - rhsa_data = requests.get( + rhsa_data = requests_session.get( f"https://access.redhat.com/hydra/rest/securitydata/cvrf/{rh_adv}.json" ).json() # nopep8 - value = rhsa_data["cvrfdoc"]["aggregate_severity"] - rhsa_aggregate_severity = VulnerabilitySeverity( - system=scoring_systems["rhas"], - value=value, - ) + + rhsa_aggregate_severities = [] + if rhsa_data.get("cvrfdoc"): + # not all RHSA errata have a corresponding CVRF document + value = rhsa_data["cvrfdoc"]["aggregate_severity"] + rhsa_aggregate_severities.append( + VulnerabilitySeverity( + system=scoring_systems["rhas"], + value=value, + ) + ) references.append( Reference( - severities=[rhsa_aggregate_severity], + severities=rhsa_aggregate_severities, url="https://access.redhat.com/errata/{}".format(rh_adv), reference_id=rh_adv, ) diff --git a/vulnerabilities/management/commands/import.py b/vulnerabilities/management/commands/import.py index 7376cf6f9..3a76fce09 100644 --- a/vulnerabilities/management/commands/import.py +++ b/vulnerabilities/management/commands/import.py @@ -22,6 +22,7 @@ # Visit https://github.com/nexB/vulnerablecode/ for support and download. from datetime import datetime +import traceback from django.core.management.base import BaseCommand from django.core.management.base import CommandError @@ -100,10 +101,21 @@ def import_data(self, names, cutoff_date): self._import_data(importers, cutoff_date) def _import_data(self, importers, cutoff_date): + failed_importers = [] + for importer in importers: self.stdout.write(f"Importing data from {importer.name}") batch_size = int(getattr(self, "batch_size", 10)) - ImportRunner(importer, batch_size).run(cutoff_date=cutoff_date) - self.stdout.write( - self.style.SUCCESS(f"Successfully imported data from {importer.name}") - ) + try: + ImportRunner(importer, batch_size).run(cutoff_date=cutoff_date) + self.stdout.write( + self.style.SUCCESS(f"Successfully imported data from {importer.name}") + ) + except Exception: + failed_importers.append(importer.name) + traceback.print_exc() + self.stdout.write( + self.style.ERROR(f"Failure to import data from {importer.name}. Continuing...") + ) + if failed_importers: + raise CommandError(f"{len(failed_importers)} failed!: {','.join(failed_importers)}") diff --git a/vulnerabilities/package_managers.py b/vulnerabilities/package_managers.py index 60c3af99b..501949201 100644 --- a/vulnerabilities/package_managers.py +++ b/vulnerabilities/package_managers.py @@ -40,12 +40,16 @@ def get(self, package_name: str) -> Set[str]: return self.cache.get(package_name, set()) +def client_session(): + return ClientSession(raise_for_status=True, trust_env=True) + + class LaunchpadVersionAPI(VersionAPI): package_type = "deb" async def load_api(self, pkg_set): - async with ClientSession(raise_for_status=True) as session: + async with client_session() as session: await asyncio.gather( *[self.set_api(pkg, session) for pkg in pkg_set if pkg not in self.cache] ) @@ -82,7 +86,7 @@ class PypiVersionAPI(VersionAPI): package_type = "pypi" async def load_api(self, pkg_set): - async with ClientSession(raise_for_status=True) as session: + async with client_session() as session: await asyncio.gather( *[self.fetch(pkg, session) for pkg in pkg_set if pkg not in self.cache] ) @@ -106,7 +110,7 @@ class CratesVersionAPI(VersionAPI): package_type = "cargo" async def load_api(self, pkg_set): - async with ClientSession(raise_for_status=True) as session: + async with client_session() as session: await asyncio.gather( *[self.fetch(pkg, session) for pkg in pkg_set if pkg not in self.cache] ) @@ -127,7 +131,7 @@ class RubyVersionAPI(VersionAPI): package_type = "gem" async def load_api(self, pkg_set): - async with ClientSession(raise_for_status=True) as session: + async with client_session() as session: await asyncio.gather( *[self.fetch(pkg, session) for pkg in pkg_set if pkg not in self.cache] ) @@ -151,7 +155,7 @@ class NpmVersionAPI(VersionAPI): package_type = "npm" async def load_api(self, pkg_set): - async with ClientSession(raise_for_status=True) as session: + async with client_session() as session: await asyncio.gather( *[self.fetch(pkg, session) for pkg in pkg_set if pkg not in self.cache] ) @@ -211,7 +215,7 @@ class MavenVersionAPI(VersionAPI): package_type = "maven" async def load_api(self, pkg_set): - async with ClientSession(raise_for_status=True) as session: + async with client_session() as session: await asyncio.gather( *[self.fetch(pkg, session) for pkg in pkg_set if pkg not in self.cache] ) @@ -267,7 +271,7 @@ class NugetVersionAPI(VersionAPI): package_type = "nuget" async def load_api(self, pkg_set): - async with ClientSession(raise_for_status=True) as session: + async with client_session() as session: await asyncio.gather( *[self.fetch(pkg, session) for pkg in pkg_set if pkg not in self.cache] ) @@ -288,9 +292,12 @@ def nuget_url(pkg_name: str) -> str: def extract_versions(resp: dict) -> Set[str]: all_versions = set() try: - for entry in resp["items"][0]["items"]: - all_versions.add(entry["catalogEntry"]["version"]) - # json response for YamlDotNet.Signed triggers this exception + for entry_group in resp["items"]: + for entry in entry_group["items"]: + all_versions.add(entry["catalogEntry"]["version"]) + # FIXME: json response for YamlDotNet.Signed triggers this exception. + # Some packages with many versions give a response of a list of endpoints. + # In such cases rather, we should collect data from those endpoints. except KeyError: pass @@ -302,7 +309,7 @@ class ComposerVersionAPI(VersionAPI): package_type = "composer" async def load_api(self, pkg_set): - async with ClientSession(raise_for_status=True) as session: + async with client_session() as session: await asyncio.gather( *[self.fetch(pkg, session) for pkg in pkg_set if pkg not in self.cache] ) @@ -340,7 +347,7 @@ class GitHubTagsAPI(VersionAPI): package_type = "github" async def load_api(self, repo_set): - async with ClientSession(raise_for_status=True) as session: + async with client_session() as session: await asyncio.gather( *[ self.fetch(owner_repo.lower(), session) @@ -360,7 +367,7 @@ async def fetch(self, owner_repo: str, session) -> None: class HexVersionAPI(VersionAPI): async def load_api(self, pkg_set): - async with ClientSession(raise_for_status=True) as session: + async with client_session() as session: await asyncio.gather( *[self.fetch(pkg, session) for pkg in pkg_set if pkg not in self.cache] ) diff --git a/vulnerabilities/severity_systems.py b/vulnerabilities/severity_systems.py index e2357856e..33594268a 100644 --- a/vulnerabilities/severity_systems.py +++ b/vulnerabilities/severity_systems.py @@ -81,4 +81,10 @@ def as_score(self, value): url="https://www.first.org/cvss/specification-document#Qualitative-Severity-Rating-Scale", notes="A textual interpretation of severity. Has values like HIGH, MODERATE etc", ), + "generic_textual": ScoringSystem( + identifier="generic_textual", + name="Generic textual severity rating", + url="", + notes="Severity for unknown scoring systems. Contains generic textual values like High, Low etc", + ), } diff --git a/vulnerabilities/tests/test_api.py b/vulnerabilities/tests/test_api.py index f95024e31..cb871f58e 100644 --- a/vulnerabilities/tests/test_api.py +++ b/vulnerabilities/tests/test_api.py @@ -258,6 +258,7 @@ def test_bulk_packages_api(self): expected_response = [ { + "purl": "pkg:deb/debian/doesnotexist@0.9.7-10?distro=jessie", "name": "doesnotexist", "namespace": "debian", "qualifiers": {"distro": "jessie"}, diff --git a/vulnerabilities/tests/test_data/kbmsr2019/test_msr_data.csv b/vulnerabilities/tests/test_data/kbmsr2019/test_msr_data.csv new file mode 100644 index 000000000..f0a35d932 --- /dev/null +++ b/vulnerabilities/tests/test_data/kbmsr2019/test_msr_data.csv @@ -0,0 +1,7 @@ +CVE-2018-11040,https://github.com/spring-projects/spring-framework,874859493bbda59739c38c7e52eb3625f247b93,pos +CVE-2013-6408,https://github.com/apache/lucene-solr,7239a57a51ea0f4d05dd330ce5e15e4f72f72747,pos +CVE-2015-6748,https://github.com/jhy/jsoup,4edb78991f8d0bf87dafde5e01ccd8922065c9b2,pos +CVE-2018-14658,https://github.com/keycloak/keycloak,a957e118e6efb35fe7ef3a62acd66341a6523cb7,pos +CVE-2017-1000355,https://github.com/jenkinsci/jenkins,701ea95a52afe53bee28f76a3f96eb0e578852e9,pos +CVE-2018-1000844,https://github.com/square/retrofit,97057aaae42e54bfbee8acfa8af7dcf37e812342,pos +HTTPCLIENT-1803,https://github.com/apache/httpcomponents-client,0554271750599756d4946c0d7ba43d04b1a7b22,pos \ No newline at end of file diff --git a/vulnerabilities/tests/test_import_runner.py b/vulnerabilities/tests/test_import_runner.py index ee6da69e9..efda749b2 100644 --- a/vulnerabilities/tests/test_import_runner.py +++ b/vulnerabilities/tests/test_import_runner.py @@ -69,9 +69,9 @@ def save(self): ADVISORIES = [ Advisory( - vulnerability_id="MOCK-CVE-2020-1337", + vulnerability_id="CVE-2020-13371337", summary="vulnerability description here", - references=[Reference(url="https://example.com/with/more/info/MOCK-CVE-2020-1337")], + references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")], impacted_package_urls=[PackageURL(name="mock-webserver", type="pypi", version="1.2.33")], resolved_package_urls=[PackageURL(name="mock-webserver", type="pypi", version="1.2.34")], ) @@ -113,11 +113,11 @@ def test_ImportRunner_new_package_and_new_vulnerability(db): assert resolved_package.vulnerabilities.count() == 1 vuln = impacted_package.vulnerabilities.first() - assert vuln.vulnerability_id == "MOCK-CVE-2020-1337" + assert vuln.vulnerability_id == "CVE-2020-13371337" vuln_refs = models.VulnerabilityReference.objects.filter(vulnerability=vuln) assert vuln_refs.count() == 1 - assert vuln_refs[0].url == "https://example.com/with/more/info/MOCK-CVE-2020-1337" + assert vuln_refs[0].url == "https://example.com/with/more/info/CVE-2020-13371337" def test_ImportRunner_existing_package_and_new_vulnerability(db): @@ -145,11 +145,11 @@ def test_ImportRunner_existing_package_and_new_vulnerability(db): impacted_package = models.PackageRelatedVulnerability.objects.filter(is_vulnerable=True)[0] vuln = impacted_package.vulnerability - assert vuln.vulnerability_id == "MOCK-CVE-2020-1337" + assert vuln.vulnerability_id == "CVE-2020-13371337" vuln_refs = models.VulnerabilityReference.objects.filter(vulnerability=vuln) assert vuln_refs.count() == 1 - assert vuln_refs[0].url == "https://example.com/with/more/info/MOCK-CVE-2020-1337" + assert vuln_refs[0].url == "https://example.com/with/more/info/CVE-2020-13371337" def test_ImportRunner_new_package_version_affected_by_existing_vulnerability(db): @@ -158,11 +158,11 @@ def test_ImportRunner_new_package_version_affected_by_existing_vulnerability(db) vulnerability that also already existed in the database. """ vuln = models.Vulnerability.objects.create( - vulnerability_id="MOCK-CVE-2020-1337", summary="vulnerability description here" + vulnerability_id="CVE-2020-13371337", summary="vulnerability description here" ) models.VulnerabilityReference.objects.create( - vulnerability=vuln, url="https://example.com/with/more/info/MOCK-CVE-2020-1337" + vulnerability=vuln, url="https://example.com/with/more/info/CVE-2020-13371337" ) models.PackageRelatedVulnerability.objects.create( vulnerability=vuln, @@ -200,7 +200,7 @@ def test_ImportRunner_new_package_version_affected_by_existing_vulnerability(db) ) assert len(qs) == 1 impacted_package = qs[0] - assert impacted_package.vulnerability.vulnerability_id == "MOCK-CVE-2020-1337" + assert impacted_package.vulnerability.vulnerability_id == "CVE-2020-13371337" # def test_ImportRunner_assumed_fixed_package_is_updated_as_impacted(db): @@ -213,11 +213,11 @@ def test_ImportRunner_new_package_version_affected_by_existing_vulnerability(db) # FIXME deleted, the referenced Package and Vulnerability are also deleted. # # vuln = models.Vulnerability.objects.create( -# vulnerability_id='MOCK-CVE-2020-1337', summary='vulnerability description here') +# vulnerability_id='CVE-2020-13371337', summary='vulnerability description here') # # models.VulnerabilityReference.objects.create( # vulnerability=vuln, -# url='https://example.com/with/more/info/MOCK-CVE-2020-1337' +# url='https://example.com/with/more/info/CVE-2020-13371337' # ) # # misclassified_package = models.Package.objects.create( @@ -255,11 +255,11 @@ def test_ImportRunner_fixed_package_version_is_added(db): A new version of a package was published that fixes a previously unresolved vulnerability. """ vuln = models.Vulnerability.objects.create( - vulnerability_id="MOCK-CVE-2020-1337", summary="vulnerability description here" + vulnerability_id="CVE-2020-13371337", summary="vulnerability description here" ) models.VulnerabilityReference.objects.create( - vulnerability=vuln, url="https://example.com/with/more/info/MOCK-CVE-2020-1337" + vulnerability=vuln, url="https://example.com/with/more/info/CVE-2020-13371337" ) models.PackageRelatedVulnerability.objects.create( vulnerability=vuln, @@ -288,7 +288,7 @@ def test_ImportRunner_fixed_package_version_is_added(db): ) assert len(qs) == 1 resolved_package = qs[0] - assert resolved_package.vulnerability.vulnerability_id == "MOCK-CVE-2020-1337" + assert resolved_package.vulnerability.vulnerability_id == "CVE-2020-13371337" def test_ImportRunner_updated_vulnerability(db): @@ -297,7 +297,7 @@ def test_ImportRunner_updated_vulnerability(db): reference. """ vuln = models.Vulnerability.objects.create( - vulnerability_id="MOCK-CVE-2020-1337", summary="temporary description" + vulnerability_id="CVE-2020-13371337", summary="temporary description" ) models.PackageRelatedVulnerability.objects.create( @@ -326,4 +326,4 @@ def test_ImportRunner_updated_vulnerability(db): vuln_refs = models.VulnerabilityReference.objects.filter(vulnerability=vuln) assert vuln_refs.count() == 1 - assert vuln_refs[0].url == "https://example.com/with/more/info/MOCK-CVE-2020-1337" + assert vuln_refs[0].url == "https://example.com/with/more/info/CVE-2020-13371337" diff --git a/vulnerabilities/tests/test_msr2019.py b/vulnerabilities/tests/test_msr2019.py new file mode 100644 index 000000000..df8956f3b --- /dev/null +++ b/vulnerabilities/tests/test_msr2019.py @@ -0,0 +1,141 @@ +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import csv +import os +from unittest import TestCase +from unittest.mock import patch + +from packageurl import PackageURL + +from vulnerabilities.data_source import Advisory +from vulnerabilities.data_source import Reference +from vulnerabilities.importers import ProjectKBMSRDataSource + + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, "test_data/kbmsr2019", "test_msr_data.csv") + + +class TestProjectKBMSRDataSource(TestCase): + def test_to_advisories(self): + with open(TEST_DATA) as f: + lines = [l for l in f.readlines()] + test_data = csv.reader(lines) + + found_advisories = ProjectKBMSRDataSource.to_advisories(test_data) + found_advisories = list(map(Advisory.normalized, found_advisories)) + expected_advisories = [ + Advisory( + summary="", + vulnerability_id="CVE-2018-11040", + impacted_package_urls=set(), + resolved_package_urls=set(), + references=[ + Reference( + reference_id="", + url="https://github.com/spring-projects/spring-framework/commit/874859493bbda59739c38c7e52eb3625f247b93", + severities=[], + ) + ], + ), + Advisory( + summary="", + vulnerability_id="CVE-2013-6408", + impacted_package_urls=set(), + resolved_package_urls=set(), + references=[ + Reference( + reference_id="", + url="https://github.com/apache/lucene-solr/commit/7239a57a51ea0f4d05dd330ce5e15e4f72f72747", + severities=[], + ) + ], + ), + Advisory( + summary="", + vulnerability_id="CVE-2015-6748", + impacted_package_urls=set(), + resolved_package_urls=set(), + references=[ + Reference( + reference_id="", + url="https://github.com/jhy/jsoup/commit/4edb78991f8d0bf87dafde5e01ccd8922065c9b2", + severities=[], + ) + ], + ), + Advisory( + summary="", + vulnerability_id="CVE-2018-14658", + impacted_package_urls=set(), + resolved_package_urls=set(), + references=[ + Reference( + reference_id="", + url="https://github.com/keycloak/keycloak/commit/a957e118e6efb35fe7ef3a62acd66341a6523cb7", + severities=[], + ) + ], + ), + Advisory( + summary="", + vulnerability_id="CVE-2017-1000355", + impacted_package_urls=set(), + resolved_package_urls=set(), + references=[ + Reference( + reference_id="", + url="https://github.com/jenkinsci/jenkins/commit/701ea95a52afe53bee28f76a3f96eb0e578852e9", + severities=[], + ) + ], + ), + Advisory( + summary="", + vulnerability_id="CVE-2018-1000844", + impacted_package_urls=set(), + resolved_package_urls=set(), + references=[ + Reference( + reference_id="", + url="https://github.com/square/retrofit/commit/97057aaae42e54bfbee8acfa8af7dcf37e812342", + severities=[], + ) + ], + ), + Advisory( + summary="", + vulnerability_id="", + impacted_package_urls=set(), + resolved_package_urls=set(), + references=[ + Reference( + reference_id="HTTPCLIENT-1803", + url="https://github.com/apache/httpcomponents-client/commit/0554271750599756d4946c0d7ba43d04b1a7b22", + severities=[], + ) + ], + ), + ] + + assert expected_advisories == found_advisories diff --git a/vulnerabilities/tests/test_nginx.py b/vulnerabilities/tests/test_nginx.py index a7f0f3837..1281bc67e 100644 --- a/vulnerabilities/tests/test_nginx.py +++ b/vulnerabilities/tests/test_nginx.py @@ -31,7 +31,6 @@ from vulnerabilities.importers.nginx import NginxDataSource from vulnerabilities.package_managers import GitHubTagsAPI -# BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data/nginx", "security_advisories.html") diff --git a/vulnerabilities/tests/test_redhat_importer.py b/vulnerabilities/tests/test_redhat_importer.py index ba7631399..967b01684 100644 --- a/vulnerabilities/tests/test_redhat_importer.py +++ b/vulnerabilities/tests/test_redhat_importer.py @@ -139,7 +139,7 @@ def test_to_advisory(self): } for adv in data: with unittest.mock.patch( - "vulnerabilities.importers.redhat.requests.get", return_value=mock_resp + "vulnerabilities.importers.redhat.requests_session.get", return_value=mock_resp ): adv = redhat.to_advisory(adv) found_advisories.append(adv) diff --git a/vulnerabilities/tests/test_upstream.py b/vulnerabilities/tests/test_upstream.py new file mode 100644 index 000000000..a875f570e --- /dev/null +++ b/vulnerabilities/tests/test_upstream.py @@ -0,0 +1,18 @@ +import pytest +from vulnerabilities import importers +from vulnerabilities.importer_yielder import IMPORTER_REGISTRY + + +@pytest.mark.webtest +@pytest.mark.parametrize( + ("data_source", "config"), + ((data["data_source"], data["data_source_cfg"]) for data in IMPORTER_REGISTRY), +) +def test_updated_advisories(data_source, config): + + if not data_source == "GitHubAPIDataSource": + data_src = getattr(importers, data_source) + data_src = data_src(batch_size=1, config=config) + with data_src: + for i in data_src.updated_advisories(): + pass diff --git a/vulnerablecode/settings.py b/vulnerablecode/settings.py index 7c829e82f..2701d89a8 100644 --- a/vulnerablecode/settings.py +++ b/vulnerablecode/settings.py @@ -26,10 +26,9 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = False +DEBUG_PROPAGATE_EXCEPTIONS = True -ALLOWED_HOSTS = [ - ".herokuapp.com", -] +ALLOWED_HOSTS = os.environ.get("VC_ALLOWED_HOSTS", "*").split(":") # Application definition