From 592e846190ac77be4d4c7c05dc697cc27f39ca1c Mon Sep 17 00:00:00 2001 From: savish Date: Thu, 18 Mar 2021 20:13:29 +0530 Subject: [PATCH 01/28] Sanity Checks for redhat import response Signed-off-by: savish --- vulnerabilities/importers/redhat.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/vulnerabilities/importers/redhat.py b/vulnerabilities/importers/redhat.py index 1fba1c484..d96036ae4 100644 --- a/vulnerabilities/importers/redhat.py +++ b/vulnerabilities/importers/redhat.py @@ -91,19 +91,24 @@ def to_advisory(advisory_data): if bugzilla: url = "https://bugzilla.redhat.com/show_bug.cgi?id={}".format(bugzilla) bugzilla_data = requests.get(f"https://bugzilla.redhat.com/rest/bug/{bugzilla}").json() - bugzilla_severity_val = bugzilla_data["bugs"][0]["severity"] - bugzilla_severity = VulnerabilitySeverity( - system=scoring_systems["rhbs"], - value=bugzilla_severity_val, - ) + if ( + bugzilla_data.get("bugs") + and len(bugzilla_data["bugs"]) + and bugzilla_data["bugs"][0].get("severity") + ): + bugzilla_severity_val = bugzilla_data["bugs"][0]["severity"] + bugzilla_severity = VulnerabilitySeverity( + system=scoring_systems["rhbs"], + value=bugzilla_severity_val, + ) - references.append( - Reference( - severities=[bugzilla_severity], - url=url, - reference_id=bugzilla, + references.append( + Reference( + severities=[bugzilla_severity], + url=url, + reference_id=bugzilla, + ) ) - ) for rh_adv in advisory_data["advisories"]: # RH provides 3 types of advisories RHSA, RHBA, RHEA. Only RHSA's contain severity score. From 1b064236c514caeb2832ac844c3be92bff0cbd6b Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Fri, 19 Mar 2021 02:05:25 +0530 Subject: [PATCH 02/28] Make sure vulnerability id is_cve or is_vulcoid This makes sure that vulnerability id supplied in alpine_linux importer is either a cve, vulcoid or empty so as to stand on the definition of vulnerability id. It could be possible to introduce a validator at the model level for the same as well using these functions Signed-off-by: Hritik Vijay --- vulnerabilities/data_source.py | 20 ++++++++++++++++++++ vulnerabilities/importers/alpine_linux.py | 4 +++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/vulnerabilities/data_source.py b/vulnerabilities/data_source.py index 28895a9cb..54db90368 100644 --- a/vulnerabilities/data_source.py +++ b/vulnerabilities/data_source.py @@ -228,6 +228,26 @@ def batch_advisories(self, advisories: List[Advisory]) -> Set[Advisory]: b, advisories = advisories[: self.batch_size], advisories[self.batch_size :] yield b + @staticmethod + def is_cve(id: str): + c = id.split("-") + if len(c) == 3 and c[0].lower() == "cve" and c[1].isdigit() and c[2].isdigit(): + return True + return False + + @staticmethod + def is_vulcoid(id: str): + c = id.split("-") + if ( + len(c) == 4 + and c[0].lower() == "vulcoid" + and c[1].isdigit() + and c[2].isdigit() + and c[3].isdigit() + ): + return True + return False + @dataclasses.dataclass class GitDataSourceConfiguration(DataSourceConfiguration): diff --git a/vulnerabilities/importers/alpine_linux.py b/vulnerabilities/importers/alpine_linux.py index d3831f47c..f114b6edc 100644 --- a/vulnerabilities/importers/alpine_linux.py +++ b/vulnerabilities/importers/alpine_linux.py @@ -193,7 +193,9 @@ def _load_advisories( impacted_package_urls=[], resolved_package_urls=resolved_purls, references=references, - vulnerability_id=vuln_ids[0] if vuln_ids[0] != "CVE-????-?????" else "", + vulnerability_id=vuln_ids[0] + if self.is_cve(vuln_ids[0]) or self.is_vulcoid(vuln_ids[0]) + else "", ) ) From 4d020a7241654e785df1738ce04ac1700180553a Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Fri, 19 Mar 2021 16:15:19 +0530 Subject: [PATCH 03/28] Use is_cve helper to validate vulnerablitiy id Signed-off-by: Hritik Vijay --- vulnerabilities/data_source.py | 25 ++++------------ vulnerabilities/importers/alpine_linux.py | 5 ++-- vulnerabilities/tests/test_import_runner.py | 32 ++++++++++----------- 3 files changed, 23 insertions(+), 39 deletions(-) diff --git a/vulnerabilities/data_source.py b/vulnerabilities/data_source.py index 54db90368..c911518ec 100644 --- a/vulnerabilities/data_source.py +++ b/vulnerabilities/data_source.py @@ -44,6 +44,7 @@ from vulnerabilities.oval_parser import OvalParser from vulnerabilities.severity_systems import ScoringSystem +from vulnerabilities.helpers import is_cve logger = logging.getLogger(__name__) @@ -88,6 +89,10 @@ class Advisory: resolved_package_urls: Iterable[PackageURL] = dataclasses.field(default_factory=list) references: List[Reference] = dataclasses.field(default_factory=list) + def __post_init__(self): + if self.vulnerability_id and not is_cve(self.vulnerability_id): + raise ValueError("CVE expected, found: {}".format(self.vulnerability_id)) + def normalized(self): impacted_package_urls = {package_url for package_url in self.impacted_package_urls} resolved_package_urls = {package_url for package_url in self.resolved_package_urls} @@ -228,26 +233,6 @@ def batch_advisories(self, advisories: List[Advisory]) -> Set[Advisory]: b, advisories = advisories[: self.batch_size], advisories[self.batch_size :] yield b - @staticmethod - def is_cve(id: str): - c = id.split("-") - if len(c) == 3 and c[0].lower() == "cve" and c[1].isdigit() and c[2].isdigit(): - return True - return False - - @staticmethod - def is_vulcoid(id: str): - c = id.split("-") - if ( - len(c) == 4 - and c[0].lower() == "vulcoid" - and c[1].isdigit() - and c[2].isdigit() - and c[3].isdigit() - ): - return True - return False - @dataclasses.dataclass class GitDataSourceConfiguration(DataSourceConfiguration): diff --git a/vulnerabilities/importers/alpine_linux.py b/vulnerabilities/importers/alpine_linux.py index f114b6edc..cd222ec46 100644 --- a/vulnerabilities/importers/alpine_linux.py +++ b/vulnerabilities/importers/alpine_linux.py @@ -38,6 +38,7 @@ from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource from vulnerabilities.data_source import Reference +from vulnerabilities.helpers import is_cve BASE_URL = "https://secdb.alpinelinux.org/" @@ -193,9 +194,7 @@ def _load_advisories( impacted_package_urls=[], resolved_package_urls=resolved_purls, references=references, - vulnerability_id=vuln_ids[0] - if self.is_cve(vuln_ids[0]) or self.is_vulcoid(vuln_ids[0]) - else "", + vulnerability_id=vuln_ids[0] if is_cve(vuln_ids[0]) else "", ) ) diff --git a/vulnerabilities/tests/test_import_runner.py b/vulnerabilities/tests/test_import_runner.py index ee6da69e9..efda749b2 100644 --- a/vulnerabilities/tests/test_import_runner.py +++ b/vulnerabilities/tests/test_import_runner.py @@ -69,9 +69,9 @@ def save(self): ADVISORIES = [ Advisory( - vulnerability_id="MOCK-CVE-2020-1337", + vulnerability_id="CVE-2020-13371337", summary="vulnerability description here", - references=[Reference(url="https://example.com/with/more/info/MOCK-CVE-2020-1337")], + references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")], impacted_package_urls=[PackageURL(name="mock-webserver", type="pypi", version="1.2.33")], resolved_package_urls=[PackageURL(name="mock-webserver", type="pypi", version="1.2.34")], ) @@ -113,11 +113,11 @@ def test_ImportRunner_new_package_and_new_vulnerability(db): assert resolved_package.vulnerabilities.count() == 1 vuln = impacted_package.vulnerabilities.first() - assert vuln.vulnerability_id == "MOCK-CVE-2020-1337" + assert vuln.vulnerability_id == "CVE-2020-13371337" vuln_refs = models.VulnerabilityReference.objects.filter(vulnerability=vuln) assert vuln_refs.count() == 1 - assert vuln_refs[0].url == "https://example.com/with/more/info/MOCK-CVE-2020-1337" + assert vuln_refs[0].url == "https://example.com/with/more/info/CVE-2020-13371337" def test_ImportRunner_existing_package_and_new_vulnerability(db): @@ -145,11 +145,11 @@ def test_ImportRunner_existing_package_and_new_vulnerability(db): impacted_package = models.PackageRelatedVulnerability.objects.filter(is_vulnerable=True)[0] vuln = impacted_package.vulnerability - assert vuln.vulnerability_id == "MOCK-CVE-2020-1337" + assert vuln.vulnerability_id == "CVE-2020-13371337" vuln_refs = models.VulnerabilityReference.objects.filter(vulnerability=vuln) assert vuln_refs.count() == 1 - assert vuln_refs[0].url == "https://example.com/with/more/info/MOCK-CVE-2020-1337" + assert vuln_refs[0].url == "https://example.com/with/more/info/CVE-2020-13371337" def test_ImportRunner_new_package_version_affected_by_existing_vulnerability(db): @@ -158,11 +158,11 @@ def test_ImportRunner_new_package_version_affected_by_existing_vulnerability(db) vulnerability that also already existed in the database. """ vuln = models.Vulnerability.objects.create( - vulnerability_id="MOCK-CVE-2020-1337", summary="vulnerability description here" + vulnerability_id="CVE-2020-13371337", summary="vulnerability description here" ) models.VulnerabilityReference.objects.create( - vulnerability=vuln, url="https://example.com/with/more/info/MOCK-CVE-2020-1337" + vulnerability=vuln, url="https://example.com/with/more/info/CVE-2020-13371337" ) models.PackageRelatedVulnerability.objects.create( vulnerability=vuln, @@ -200,7 +200,7 @@ def test_ImportRunner_new_package_version_affected_by_existing_vulnerability(db) ) assert len(qs) == 1 impacted_package = qs[0] - assert impacted_package.vulnerability.vulnerability_id == "MOCK-CVE-2020-1337" + assert impacted_package.vulnerability.vulnerability_id == "CVE-2020-13371337" # def test_ImportRunner_assumed_fixed_package_is_updated_as_impacted(db): @@ -213,11 +213,11 @@ def test_ImportRunner_new_package_version_affected_by_existing_vulnerability(db) # FIXME deleted, the referenced Package and Vulnerability are also deleted. # # vuln = models.Vulnerability.objects.create( -# vulnerability_id='MOCK-CVE-2020-1337', summary='vulnerability description here') +# vulnerability_id='CVE-2020-13371337', summary='vulnerability description here') # # models.VulnerabilityReference.objects.create( # vulnerability=vuln, -# url='https://example.com/with/more/info/MOCK-CVE-2020-1337' +# url='https://example.com/with/more/info/CVE-2020-13371337' # ) # # misclassified_package = models.Package.objects.create( @@ -255,11 +255,11 @@ def test_ImportRunner_fixed_package_version_is_added(db): A new version of a package was published that fixes a previously unresolved vulnerability. """ vuln = models.Vulnerability.objects.create( - vulnerability_id="MOCK-CVE-2020-1337", summary="vulnerability description here" + vulnerability_id="CVE-2020-13371337", summary="vulnerability description here" ) models.VulnerabilityReference.objects.create( - vulnerability=vuln, url="https://example.com/with/more/info/MOCK-CVE-2020-1337" + vulnerability=vuln, url="https://example.com/with/more/info/CVE-2020-13371337" ) models.PackageRelatedVulnerability.objects.create( vulnerability=vuln, @@ -288,7 +288,7 @@ def test_ImportRunner_fixed_package_version_is_added(db): ) assert len(qs) == 1 resolved_package = qs[0] - assert resolved_package.vulnerability.vulnerability_id == "MOCK-CVE-2020-1337" + assert resolved_package.vulnerability.vulnerability_id == "CVE-2020-13371337" def test_ImportRunner_updated_vulnerability(db): @@ -297,7 +297,7 @@ def test_ImportRunner_updated_vulnerability(db): reference. """ vuln = models.Vulnerability.objects.create( - vulnerability_id="MOCK-CVE-2020-1337", summary="temporary description" + vulnerability_id="CVE-2020-13371337", summary="temporary description" ) models.PackageRelatedVulnerability.objects.create( @@ -326,4 +326,4 @@ def test_ImportRunner_updated_vulnerability(db): vuln_refs = models.VulnerabilityReference.objects.filter(vulnerability=vuln) assert vuln_refs.count() == 1 - assert vuln_refs[0].url == "https://example.com/with/more/info/MOCK-CVE-2020-1337" + assert vuln_refs[0].url == "https://example.com/with/more/info/CVE-2020-13371337" From f06e1b2f5432a807ac8d6649ab25e5c78ace4bcc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 19 Mar 2021 21:55:38 +0000 Subject: [PATCH 04/28] Bump djangorestframework from 3.11.0 to 3.11.2 Bumps [djangorestframework](https://github.com/encode/django-rest-framework) from 3.11.0 to 3.11.2. - [Release notes](https://github.com/encode/django-rest-framework/releases) - [Commits](https://github.com/encode/django-rest-framework/compare/3.11.0...3.11.2) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 37d2014cf..705ca9c65 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ dephell-specifier==0.2.1 dj-database-url==0.4.2 Django==3.0.7 django-filter==2.2.0 -djangorestframework==3.11.0 +djangorestframework==3.11.2 django-widget-tweaks==1.4.8 drf-spectacular==0.13.0 gunicorn==19.7.1 From d8d07abd5592b9e8051b6217bfe7213455f888c1 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Mon, 15 Mar 2021 13:50:13 +0530 Subject: [PATCH 05/28] Fix various importer errors - Update debian importer's schema validation - Add tests for msr2019 importer. Signed-off-by: Shivam Sandbhor --- vulnerabilities/importers/debian.py | 1 + .../importers/project_kb_msr2019.py | 4 +- .../test_data/kbmsr2019/test_msr_data.csv | 7 + vulnerabilities/tests/test_msr2019.py | 141 ++++++++++++++++++ vulnerabilities/tests/test_nginx.py | 1 - 5 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 vulnerabilities/tests/test_data/kbmsr2019/test_msr_data.csv create mode 100644 vulnerabilities/tests/test_msr2019.py diff --git a/vulnerabilities/importers/debian.py b/vulnerabilities/importers/debian.py index ae9004f37..28d499980 100644 --- a/vulnerabilities/importers/debian.py +++ b/vulnerabilities/importers/debian.py @@ -45,6 +45,7 @@ def validate_schema(advisory_dict): deb_versions = [ "bullseye", + "bullseye-security", "buster", "buster-security", "sid", diff --git a/vulnerabilities/importers/project_kb_msr2019.py b/vulnerabilities/importers/project_kb_msr2019.py index 156cfb82f..b9f797c48 100644 --- a/vulnerabilities/importers/project_kb_msr2019.py +++ b/vulnerabilities/importers/project_kb_msr2019.py @@ -47,7 +47,7 @@ class ProjectKBMSRDataSource(DataSource): CONFIG_CLASS = ProjectKBDataSourceConfiguration - url = "https://raw.githubusercontent.com/SAP/project-kb/master/MSR2019/dataset/vulas_db_msr2019_release.csv" # nopep8 + url = "https://raw.githubusercontent.com/SAP/project-kb/master/MSR2019/dataset/vulas_db_msr2019_release.csv" def updated_advisories(self): if create_etag(data_src=self, url=self.url, etag_key="ETag"): @@ -82,7 +82,7 @@ def to_advisories(csv_reader): summary="", impacted_package_urls=[], references=[reference], - cve_id=vuln_id, + vulnerability_id=vuln_id, ) ) diff --git a/vulnerabilities/tests/test_data/kbmsr2019/test_msr_data.csv b/vulnerabilities/tests/test_data/kbmsr2019/test_msr_data.csv new file mode 100644 index 000000000..f0a35d932 --- /dev/null +++ b/vulnerabilities/tests/test_data/kbmsr2019/test_msr_data.csv @@ -0,0 +1,7 @@ +CVE-2018-11040,https://github.com/spring-projects/spring-framework,874859493bbda59739c38c7e52eb3625f247b93,pos +CVE-2013-6408,https://github.com/apache/lucene-solr,7239a57a51ea0f4d05dd330ce5e15e4f72f72747,pos +CVE-2015-6748,https://github.com/jhy/jsoup,4edb78991f8d0bf87dafde5e01ccd8922065c9b2,pos +CVE-2018-14658,https://github.com/keycloak/keycloak,a957e118e6efb35fe7ef3a62acd66341a6523cb7,pos +CVE-2017-1000355,https://github.com/jenkinsci/jenkins,701ea95a52afe53bee28f76a3f96eb0e578852e9,pos +CVE-2018-1000844,https://github.com/square/retrofit,97057aaae42e54bfbee8acfa8af7dcf37e812342,pos +HTTPCLIENT-1803,https://github.com/apache/httpcomponents-client,0554271750599756d4946c0d7ba43d04b1a7b22,pos \ No newline at end of file diff --git a/vulnerabilities/tests/test_msr2019.py b/vulnerabilities/tests/test_msr2019.py new file mode 100644 index 000000000..df8956f3b --- /dev/null +++ b/vulnerabilities/tests/test_msr2019.py @@ -0,0 +1,141 @@ +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import csv +import os +from unittest import TestCase +from unittest.mock import patch + +from packageurl import PackageURL + +from vulnerabilities.data_source import Advisory +from vulnerabilities.data_source import Reference +from vulnerabilities.importers import ProjectKBMSRDataSource + + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, "test_data/kbmsr2019", "test_msr_data.csv") + + +class TestProjectKBMSRDataSource(TestCase): + def test_to_advisories(self): + with open(TEST_DATA) as f: + lines = [l for l in f.readlines()] + test_data = csv.reader(lines) + + found_advisories = ProjectKBMSRDataSource.to_advisories(test_data) + found_advisories = list(map(Advisory.normalized, found_advisories)) + expected_advisories = [ + Advisory( + summary="", + vulnerability_id="CVE-2018-11040", + impacted_package_urls=set(), + resolved_package_urls=set(), + references=[ + Reference( + reference_id="", + url="https://github.com/spring-projects/spring-framework/commit/874859493bbda59739c38c7e52eb3625f247b93", + severities=[], + ) + ], + ), + Advisory( + summary="", + vulnerability_id="CVE-2013-6408", + impacted_package_urls=set(), + resolved_package_urls=set(), + references=[ + Reference( + reference_id="", + url="https://github.com/apache/lucene-solr/commit/7239a57a51ea0f4d05dd330ce5e15e4f72f72747", + severities=[], + ) + ], + ), + Advisory( + summary="", + vulnerability_id="CVE-2015-6748", + impacted_package_urls=set(), + resolved_package_urls=set(), + references=[ + Reference( + reference_id="", + url="https://github.com/jhy/jsoup/commit/4edb78991f8d0bf87dafde5e01ccd8922065c9b2", + severities=[], + ) + ], + ), + Advisory( + summary="", + vulnerability_id="CVE-2018-14658", + impacted_package_urls=set(), + resolved_package_urls=set(), + references=[ + Reference( + reference_id="", + url="https://github.com/keycloak/keycloak/commit/a957e118e6efb35fe7ef3a62acd66341a6523cb7", + severities=[], + ) + ], + ), + Advisory( + summary="", + vulnerability_id="CVE-2017-1000355", + impacted_package_urls=set(), + resolved_package_urls=set(), + references=[ + Reference( + reference_id="", + url="https://github.com/jenkinsci/jenkins/commit/701ea95a52afe53bee28f76a3f96eb0e578852e9", + severities=[], + ) + ], + ), + Advisory( + summary="", + vulnerability_id="CVE-2018-1000844", + impacted_package_urls=set(), + resolved_package_urls=set(), + references=[ + Reference( + reference_id="", + url="https://github.com/square/retrofit/commit/97057aaae42e54bfbee8acfa8af7dcf37e812342", + severities=[], + ) + ], + ), + Advisory( + summary="", + vulnerability_id="", + impacted_package_urls=set(), + resolved_package_urls=set(), + references=[ + Reference( + reference_id="HTTPCLIENT-1803", + url="https://github.com/apache/httpcomponents-client/commit/0554271750599756d4946c0d7ba43d04b1a7b22", + severities=[], + ) + ], + ), + ] + + assert expected_advisories == found_advisories diff --git a/vulnerabilities/tests/test_nginx.py b/vulnerabilities/tests/test_nginx.py index a7f0f3837..1281bc67e 100644 --- a/vulnerabilities/tests/test_nginx.py +++ b/vulnerabilities/tests/test_nginx.py @@ -31,7 +31,6 @@ from vulnerabilities.importers.nginx import NginxDataSource from vulnerabilities.package_managers import GitHubTagsAPI -# BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data/nginx", "security_advisories.html") From da4325b1145af17ff4fdd6c6e8db73e1ddb00010 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Mar 2021 23:19:26 +0000 Subject: [PATCH 06/28] Bump pyyaml from 5.3.1 to 5.4 Bumps [pyyaml](https://github.com/yaml/pyyaml) from 5.3.1 to 5.4. - [Release notes](https://github.com/yaml/pyyaml/releases) - [Changelog](https://github.com/yaml/pyyaml/blob/master/CHANGES) - [Commits](https://github.com/yaml/pyyaml/compare/5.3.1...5.4) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 37d2014cf..a3a24af4d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -54,5 +54,5 @@ whitenoise==5.0.1 zipp==0.6.0 requests==2.23.0 toml==0.10.2 -PyYAML==5.3.1 +PyYAML==5.4 freezegun==1.1.0 \ No newline at end of file From fdd029b6f90f2f9a5d799cb5e6da40881595a77f Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Fri, 26 Mar 2021 09:43:03 +0530 Subject: [PATCH 07/28] Update to django 3.0.13 Signed-off-by: Shivam Sandbhor --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a3a24af4d..f785afcfc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ contextlib2==0.5.5 decorator==4.4.2 dephell-specifier==0.2.1 dj-database-url==0.4.2 -Django==3.0.7 +Django==3.0.13 django-filter==2.2.0 djangorestframework==3.11.0 django-widget-tweaks==1.4.8 From 4aab987ffc908c85f78d934a02420da28b59ec64 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 26 Mar 2021 04:16:15 +0000 Subject: [PATCH 08/28] Bump lxml from 4.6.2 to 4.6.3 Bumps [lxml](https://github.com/lxml/lxml) from 4.6.2 to 4.6.3. - [Release notes](https://github.com/lxml/lxml/releases) - [Changelog](https://github.com/lxml/lxml/blob/master/CHANGES.txt) - [Commits](https://github.com/lxml/lxml/compare/lxml-4.6.2...lxml-4.6.3) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a9f514698..1c3a89a71 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ importlib-metadata==1.3.0 ipython==7.13.0 ipython-genutils==0.2.0 jedi==0.17.0 -lxml==4.6.2 +lxml==4.6.3 more-itertools==8.0.2 packageurl-python==0.9.3 packaging==19.2 From adf86fdc2a42f93a19f71f40c2ba7350d9d7c220 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Fri, 26 Mar 2021 15:14:11 +0530 Subject: [PATCH 09/28] Collect more data at NugetVersionApi - Now the class handles paginated results - Added a detailed FIXME comment about an edge case Signed-off-by: Shivam Sandbhor --- vulnerabilities/package_managers.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/vulnerabilities/package_managers.py b/vulnerabilities/package_managers.py index 60c3af99b..5778a3ee6 100644 --- a/vulnerabilities/package_managers.py +++ b/vulnerabilities/package_managers.py @@ -288,9 +288,12 @@ def nuget_url(pkg_name: str) -> str: def extract_versions(resp: dict) -> Set[str]: all_versions = set() try: - for entry in resp["items"][0]["items"]: - all_versions.add(entry["catalogEntry"]["version"]) - # json response for YamlDotNet.Signed triggers this exception + for entry_group in resp["items"]: + for entry in entry_group["items"]: + all_versions.add(entry["catalogEntry"]["version"]) + # FIXME: json response for YamlDotNet.Signed triggers this exception. + # Some packages with many versions give a response of a list of endpoints. + # In such cases rather, we should collect data from those endpoints. except KeyError: pass From 27ceb23b4b6e70c266fbf0f899f32b3ba7559204 Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Sat, 27 Mar 2021 03:45:54 +0530 Subject: [PATCH 10/28] Add me to AUTHORS I hope no review is required :p Signed-off-by: Hritik Vijay --- AUTHORS.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 039459f93..baf9bc569 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -11,4 +11,5 @@ The following organizations or individuals have contributed to this repo: - Islam Elhakmi @EslamHiko - Edoardo Lanzini @elanzini - Navonil Das @NavonilDas -- Tushar Upadhyay @tushar912 \ No newline at end of file +- Tushar Upadhyay @tushar912 +- Hritik Vijay @hritik14 From 383e87c0bad3a5d619b0a82cd067171bb8e85add Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rolf=20Schr=C3=B6der?= Date: Sat, 27 Mar 2021 13:32:16 +0100 Subject: [PATCH 11/28] Update nix deps. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rolf Schröder --- etc/nix/flake.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/nix/flake.nix b/etc/nix/flake.nix index 10cc4c7cd..ca29e3bc7 100644 --- a/etc/nix/flake.nix +++ b/etc/nix/flake.nix @@ -53,9 +53,9 @@ # mach-nix release) is usually insufficient. Use # ./get-latest-pypi-deps-db.sh to obtain the data rev & hash. pypiDataRev = - "499750266bb4b2840cbe856c2cc0e3297685e362"; # 2021-03-06T08:13:08Z + "e9b0fc6b92cd6efbca7ba3b3d4a551bcc13a73c5"; # 2021-03-27T08:13:04Z pypiDataSha256 = - "188g24k8pk4lgqybywimkvwjwh8014v6l2mrkvzv309882i9p5gc"; + "1ssa48l2iz8kncby1gfrbds79mg114dkhpxrridwcq6q2c37p62s"; }); in { From 8c22bee6cc9106a525592079ec1612ac6ec647be Mon Sep 17 00:00:00 2001 From: Pierre Tardy Date: Fri, 26 Mar 2021 18:40:24 +0100 Subject: [PATCH 12/28] enable configuration of allowed host Signed-off-by: Pierre Tardy --- README.rst | 5 +++++ vulnerablecode/settings.py | 4 +--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index d328634c9..6bd04c94a 100644 --- a/README.rst +++ b/README.rst @@ -160,6 +160,11 @@ for this purpose:: SECRET_KEY=$(python -c "from django.core.management import utils; print(utils.get_random_secret_key())") +You will also need to setup the VC_ALLOWED_HOSTS environment variable to match the hostname where the app is deployed:: + + VC_ALLOWED_HOSTS=vulnerablecode.your.domain.example.com + +You can specify several host by separating them with a colon `:` Using Nix ~~~~~~~~~ diff --git a/vulnerablecode/settings.py b/vulnerablecode/settings.py index 7c829e82f..ae9cf8384 100644 --- a/vulnerablecode/settings.py +++ b/vulnerablecode/settings.py @@ -27,9 +27,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = False -ALLOWED_HOSTS = [ - ".herokuapp.com", -] +ALLOWED_HOSTS = os.environ.get("VC_ALLOWED_HOSTS", "*").split(":") # Application definition From e1b6ab959252e6e069d3a8ce958df1cd7b965fdb Mon Sep 17 00:00:00 2001 From: Pierre Tardy Date: Mon, 29 Mar 2021 20:29:52 +0200 Subject: [PATCH 13/28] import: continue upon failure If there is an operational error on one of the importer, it is better to still run the other importers Signed-off-by: Pierre Tardy --- vulnerabilities/management/commands/import.py | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/vulnerabilities/management/commands/import.py b/vulnerabilities/management/commands/import.py index 7376cf6f9..3a76fce09 100644 --- a/vulnerabilities/management/commands/import.py +++ b/vulnerabilities/management/commands/import.py @@ -22,6 +22,7 @@ # Visit https://github.com/nexB/vulnerablecode/ for support and download. from datetime import datetime +import traceback from django.core.management.base import BaseCommand from django.core.management.base import CommandError @@ -100,10 +101,21 @@ def import_data(self, names, cutoff_date): self._import_data(importers, cutoff_date) def _import_data(self, importers, cutoff_date): + failed_importers = [] + for importer in importers: self.stdout.write(f"Importing data from {importer.name}") batch_size = int(getattr(self, "batch_size", 10)) - ImportRunner(importer, batch_size).run(cutoff_date=cutoff_date) - self.stdout.write( - self.style.SUCCESS(f"Successfully imported data from {importer.name}") - ) + try: + ImportRunner(importer, batch_size).run(cutoff_date=cutoff_date) + self.stdout.write( + self.style.SUCCESS(f"Successfully imported data from {importer.name}") + ) + except Exception: + failed_importers.append(importer.name) + traceback.print_exc() + self.stdout.write( + self.style.ERROR(f"Failure to import data from {importer.name}. Continuing...") + ) + if failed_importers: + raise CommandError(f"{len(failed_importers)} failed!: {','.join(failed_importers)}") From b78cd91ce8cca359442aa633a25858b0125a9ea2 Mon Sep 17 00:00:00 2001 From: Pierre Tardy Date: Mon, 29 Mar 2021 13:54:09 +0200 Subject: [PATCH 14/28] DEBUG_PROPAGATE_EXCEPTIONS = True This allows to print stack traces on the stdout Best practice would be to integrate with sentry, but at least print the stack trace is helpful when trying to deploy this Signed-off-by: Pierre Tardy --- vulnerablecode/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vulnerablecode/settings.py b/vulnerablecode/settings.py index 7c829e82f..c21db3ca3 100644 --- a/vulnerablecode/settings.py +++ b/vulnerablecode/settings.py @@ -26,6 +26,7 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = False +DEBUG_PROPAGATE_EXCEPTIONS = True ALLOWED_HOSTS = [ ".herokuapp.com", From c2e1b23b4641ebe8678e990e434555ce222eee4b Mon Sep 17 00:00:00 2001 From: Pierre Tardy Date: Mon, 29 Mar 2021 14:00:47 +0200 Subject: [PATCH 15/28] Enforce static collection in dockerfile Signed-off-by: Pierre Tardy --- Dockerfile | 3 ++- README.rst | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3cc935436..c4308986e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,8 @@ ENV PYTHONUNBUFFERED 1 RUN mkdir /vulnerablecode WORKDIR /vulnerablecode ADD . /vulnerablecode/ -RUN pip install -r requirements.txt +RUN pip install -r requirements.txt && \ + DJANGO_DEV=1 python manage.py collectstatic LABEL "base_image": "pkg:docker/python@sha256%3Ae9b7e3b4e9569808066c5901b8a9ad315a9f14ae8d3949ece22ae339fff2cad0" LABEL "dockerfile_url": "https://github.com/nexB/vulnerablecode/blob/develop/Dockerfile" diff --git a/README.rst b/README.rst index d328634c9..87428ea54 100644 --- a/README.rst +++ b/README.rst @@ -141,11 +141,12 @@ On Debian-based distros, these can be installed with:: **Application dependencies** -Create a virtualenv, install dependencies, and run the database migrations:: +Create a virtualenv, install dependencies, generate static files and run the database migrations:: python3 -m venv venv source venv/bin/activate pip install -r requirements.txt + DJANGO_DEV=1 python manage.py collectstatic DJANGO_DEV=1 python manage.py migrate The environment variable ``DJANGO_DEV`` is used to load settings suitable for From ca32550c15a9e1327d88b78f351ce05c6653de5b Mon Sep 17 00:00:00 2001 From: Pierre Tardy Date: Mon, 29 Mar 2021 14:31:08 +0200 Subject: [PATCH 16/28] enable aiohttp client to trust environement for proxy Signed-off-by: Pierre Tardy --- vulnerabilities/package_managers.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/vulnerabilities/package_managers.py b/vulnerabilities/package_managers.py index 5778a3ee6..501949201 100644 --- a/vulnerabilities/package_managers.py +++ b/vulnerabilities/package_managers.py @@ -40,12 +40,16 @@ def get(self, package_name: str) -> Set[str]: return self.cache.get(package_name, set()) +def client_session(): + return ClientSession(raise_for_status=True, trust_env=True) + + class LaunchpadVersionAPI(VersionAPI): package_type = "deb" async def load_api(self, pkg_set): - async with ClientSession(raise_for_status=True) as session: + async with client_session() as session: await asyncio.gather( *[self.set_api(pkg, session) for pkg in pkg_set if pkg not in self.cache] ) @@ -82,7 +86,7 @@ class PypiVersionAPI(VersionAPI): package_type = "pypi" async def load_api(self, pkg_set): - async with ClientSession(raise_for_status=True) as session: + async with client_session() as session: await asyncio.gather( *[self.fetch(pkg, session) for pkg in pkg_set if pkg not in self.cache] ) @@ -106,7 +110,7 @@ class CratesVersionAPI(VersionAPI): package_type = "cargo" async def load_api(self, pkg_set): - async with ClientSession(raise_for_status=True) as session: + async with client_session() as session: await asyncio.gather( *[self.fetch(pkg, session) for pkg in pkg_set if pkg not in self.cache] ) @@ -127,7 +131,7 @@ class RubyVersionAPI(VersionAPI): package_type = "gem" async def load_api(self, pkg_set): - async with ClientSession(raise_for_status=True) as session: + async with client_session() as session: await asyncio.gather( *[self.fetch(pkg, session) for pkg in pkg_set if pkg not in self.cache] ) @@ -151,7 +155,7 @@ class NpmVersionAPI(VersionAPI): package_type = "npm" async def load_api(self, pkg_set): - async with ClientSession(raise_for_status=True) as session: + async with client_session() as session: await asyncio.gather( *[self.fetch(pkg, session) for pkg in pkg_set if pkg not in self.cache] ) @@ -211,7 +215,7 @@ class MavenVersionAPI(VersionAPI): package_type = "maven" async def load_api(self, pkg_set): - async with ClientSession(raise_for_status=True) as session: + async with client_session() as session: await asyncio.gather( *[self.fetch(pkg, session) for pkg in pkg_set if pkg not in self.cache] ) @@ -267,7 +271,7 @@ class NugetVersionAPI(VersionAPI): package_type = "nuget" async def load_api(self, pkg_set): - async with ClientSession(raise_for_status=True) as session: + async with client_session() as session: await asyncio.gather( *[self.fetch(pkg, session) for pkg in pkg_set if pkg not in self.cache] ) @@ -305,7 +309,7 @@ class ComposerVersionAPI(VersionAPI): package_type = "composer" async def load_api(self, pkg_set): - async with ClientSession(raise_for_status=True) as session: + async with client_session() as session: await asyncio.gather( *[self.fetch(pkg, session) for pkg in pkg_set if pkg not in self.cache] ) @@ -343,7 +347,7 @@ class GitHubTagsAPI(VersionAPI): package_type = "github" async def load_api(self, repo_set): - async with ClientSession(raise_for_status=True) as session: + async with client_session() as session: await asyncio.gather( *[ self.fetch(owner_repo.lower(), session) @@ -363,7 +367,7 @@ async def fetch(self, owner_repo: str, session) -> None: class HexVersionAPI(VersionAPI): async def load_api(self, pkg_set): - async with ClientSession(raise_for_status=True) as session: + async with client_session() as session: await asyncio.gather( *[self.fetch(pkg, session) for pkg in pkg_set if pkg not in self.cache] ) From d96fe5b07c7f91517c5e9dde35329d2d2a0dcaa3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Mar 2021 23:28:25 +0000 Subject: [PATCH 17/28] Bump pygments from 2.6.1 to 2.7.4 Bumps [pygments](https://github.com/pygments/pygments) from 2.6.1 to 2.7.4. - [Release notes](https://github.com/pygments/pygments/releases) - [Changelog](https://github.com/pygments/pygments/blob/master/CHANGES) - [Commits](https://github.com/pygments/pygments/compare/2.6.1...2.7.4) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a9f514698..a8a1da1d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,7 +34,7 @@ ptyprocess==0.6.0 py==1.8.0 pycparser==2.20 pygit2==1.5.0 -Pygments==2.6.1 +Pygments==2.7.4 pyparsing==2.4.5 pytest==5.3.2 django-widget-tweaks==1.4.8 From 8b6aae018694b557a17fd42aff42796ec35061d8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 30 Mar 2021 04:06:32 +0000 Subject: [PATCH 18/28] Bump lxml from 4.6.2 to 4.6.3 Bumps [lxml](https://github.com/lxml/lxml) from 4.6.2 to 4.6.3. - [Release notes](https://github.com/lxml/lxml/releases) - [Changelog](https://github.com/lxml/lxml/blob/master/CHANGES.txt) - [Commits](https://github.com/lxml/lxml/compare/lxml-4.6.2...lxml-4.6.3) Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a8a1da1d1..6d45f57bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ importlib-metadata==1.3.0 ipython==7.13.0 ipython-genutils==0.2.0 jedi==0.17.0 -lxml==4.6.2 +lxml==4.6.3 more-itertools==8.0.2 packageurl-python==0.9.3 packaging==19.2 From 0b7dc3d263a1a0dddde0e50dff1fa9f21bf68a08 Mon Sep 17 00:00:00 2001 From: Hritik Vijay <7457065+Hritik14@users.noreply.github.com> Date: Tue, 30 Mar 2021 09:39:45 +0530 Subject: [PATCH 19/28] Fix istio (#395) * Fix https://github.com/nexB/vulnerablecode/issues/394 Signed-off-by: Hritik Vijay --- vulnerabilities/importers/istio.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vulnerabilities/importers/istio.py b/vulnerabilities/importers/istio.py index fa0f1f903..5e29e739b 100644 --- a/vulnerabilities/importers/istio.py +++ b/vulnerabilities/importers/istio.py @@ -31,6 +31,8 @@ from vulnerabilities.data_source import Advisory, GitDataSource, Reference from vulnerabilities.package_managers import GitHubTagsAPI +is_release = re.compile(r"^[\d.]+$", re.IGNORECASE).match + class IstioDataSource(GitDataSource): def __enter__(self): @@ -47,7 +49,7 @@ def set_api(self): asyncio.run(self.version_api.load_api(["istio/istio"])) def updated_advisories(self) -> Set[Advisory]: - files = self._updated_files + files = self._added_files.union(self._updated_files) advisories = [] for f in files: processed_data = self.process_file(f) @@ -195,5 +197,3 @@ def get_data_from_md(self, path): with open(path) as f: yaml_lines = self.get_yaml_lines(f) return self.get_data_from_yaml_lines(yaml_lines) - - is_release = re.compile(r"^[\d.]+$", re.IGNORECASE).match From eb524148afa1a9c23f2eb4cdec336879b00d794c Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Thu, 1 Apr 2021 20:40:36 +0530 Subject: [PATCH 20/28] Fix bulk api in case purl does not exist Signed-off-by: Shivam Sandbhor --- vulnerabilities/api.py | 2 ++ vulnerabilities/tests/test_api.py | 1 + 2 files changed, 3 insertions(+) diff --git a/vulnerabilities/api.py b/vulnerabilities/api.py index 07efcfc55..cb3fc1756 100644 --- a/vulnerabilities/api.py +++ b/vulnerabilities/api.py @@ -152,6 +152,7 @@ def bulk_search(self, request): ) for purl in request.data["purls"]: try: + purl_string = purl purl = PackageURL.from_string(purl).to_dict() except ValueError as ve: return Response(status=400, data={"Error": f"Invalid Package URL: {purl}"}) @@ -165,6 +166,7 @@ def bulk_search(self, request): purl_response = purl purl_response["unresolved_vulnerabilities"] = [] purl_response["resolved_vulnerabilities"] = [] + purl_response["purl"] = purl_string response.append(purl_response) return Response(response) diff --git a/vulnerabilities/tests/test_api.py b/vulnerabilities/tests/test_api.py index f95024e31..cb871f58e 100644 --- a/vulnerabilities/tests/test_api.py +++ b/vulnerabilities/tests/test_api.py @@ -258,6 +258,7 @@ def test_bulk_packages_api(self): expected_response = [ { + "purl": "pkg:deb/debian/doesnotexist@0.9.7-10?distro=jessie", "name": "doesnotexist", "namespace": "debian", "qualifiers": {"distro": "jessie"}, From 2684f0564ba408d87cd0f23d58dc35351a0b6997 Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Tue, 30 Mar 2021 03:20:12 +0530 Subject: [PATCH 21/28] requests_with_5xx_retry: Retry on 5xx errors Introduced and used a helper function for retries on 5xx errors. This is important and some servers like bugzilla.redhat.com return 502 Proxy Error which was the cause of https://github.com/nexB/vulnerablecode/issues/398 A ticket has been raised in RedHat here https://redhat.service-now.com/help?id=rh_ticket&table=sc_req_item&sys_id=278239541b1ba010477e43fccd4bcb4a Signed-off-by: Hritik Vijay --- vulnerabilities/helpers.py | 19 ++++ vulnerabilities/importers/redhat.py | 162 +++++++++++++++------------- 2 files changed, 108 insertions(+), 73 deletions(-) diff --git a/vulnerabilities/helpers.py b/vulnerabilities/helpers.py index 36bf7c865..77376ee94 100644 --- a/vulnerabilities/helpers.py +++ b/vulnerabilities/helpers.py @@ -25,6 +25,7 @@ import requests import toml +import urllib3 import yaml # TODO add logging here @@ -79,3 +80,21 @@ def create_etag(data_src, url, etag_key): is_cve = re.compile(r"CVE-\d+-\d+", re.IGNORECASE).match + + +def requests_with_5xx_retry(max_retries=5, backoff_factor=0.5): + """ + Returns a requests sessions which retries on 5xx errors with + a backoff_factor + """ + retries = urllib3.util.Retry( + total=max_retries, + backoff_factor=backoff_factor, + raise_on_status=True, + status_forcelist=range(500, 600, 1), + ) + adapter = requests.adapters.HTTPAdapter(max_retries=retries) + session = requests.Session() + session.mount("https://", adapter) + session.mount("http://", adapter) + return session diff --git a/vulnerabilities/importers/redhat.py b/vulnerabilities/importers/redhat.py index d96036ae4..a591d3e72 100644 --- a/vulnerabilities/importers/redhat.py +++ b/vulnerabilities/importers/redhat.py @@ -20,17 +20,20 @@ # VulnerableCode is a free software code from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. -from packageurl import PackageURL import requests +from packageurl import PackageURL from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource from vulnerabilities.data_source import DataSourceConfiguration from vulnerabilities.data_source import Reference from vulnerabilities.data_source import VulnerabilitySeverity +from vulnerabilities.helpers import requests_with_5xx_retry from vulnerabilities.severity_systems import scoring_systems +import pdb + class RedhatDataSource(DataSource): CONFIG_CLASS = DataSourceConfiguration @@ -43,6 +46,9 @@ def updated_advisories(self): return self.batch_advisories(processed_advisories) +requests_session = requests_with_5xx_retry(max_retries=5, backoff_factor=1) + + def fetch(): """ Return a list of CVE data mappings fetched from the RedHat API. @@ -58,7 +64,7 @@ def fetch(): current_url = url_template.format(page_no) try: print(f"Fetching: {current_url}") - response = requests.get(current_url) + response = requests_session.get(current_url) if response.status_code != requests.codes.ok: # TODO: log me print(f"Failed to fetch results from {current_url}") @@ -79,88 +85,98 @@ def fetch(): def to_advisory(advisory_data): - affected_purls = [] - if advisory_data.get("affected_packages"): - for rpm in advisory_data["affected_packages"]: - purl = rpm_to_purl(rpm) - if purl: - affected_purls.append(purl) - - references = [] - bugzilla = advisory_data.get("bugzilla") - if bugzilla: - url = "https://bugzilla.redhat.com/show_bug.cgi?id={}".format(bugzilla) - bugzilla_data = requests.get(f"https://bugzilla.redhat.com/rest/bug/{bugzilla}").json() - if ( - bugzilla_data.get("bugs") - and len(bugzilla_data["bugs"]) - and bugzilla_data["bugs"][0].get("severity") - ): - bugzilla_severity_val = bugzilla_data["bugs"][0]["severity"] - bugzilla_severity = VulnerabilitySeverity( - system=scoring_systems["rhbs"], - value=bugzilla_severity_val, - ) + try: + affected_purls = [] + if advisory_data.get("affected_packages"): + for rpm in advisory_data["affected_packages"]: + purl = rpm_to_purl(rpm) + if purl: + affected_purls.append(purl) + + references = [] + bugzilla = advisory_data.get("bugzilla") + if bugzilla: + url = "https://bugzilla.redhat.com/show_bug.cgi?id={}".format(bugzilla) + bugzilla_data = requests_session.get( + f"https://bugzilla.redhat.com/rest/bug/{bugzilla}" + ).json() + if ( + bugzilla_data.get("bugs") + and len(bugzilla_data["bugs"]) + and bugzilla_data["bugs"][0].get("severity") + ): + bugzilla_severity_val = bugzilla_data["bugs"][0]["severity"] + bugzilla_severity = VulnerabilitySeverity( + system=scoring_systems["rhbs"], + value=bugzilla_severity_val, + ) - references.append( - Reference( - severities=[bugzilla_severity], - url=url, - reference_id=bugzilla, + references.append( + Reference( + severities=[bugzilla_severity], + url=url, + reference_id=bugzilla, + ) ) - ) - for rh_adv in advisory_data["advisories"]: - # RH provides 3 types of advisories RHSA, RHBA, RHEA. Only RHSA's contain severity score. - # See https://access.redhat.com/articles/2130961 for more details. - - if "RHSA" in rh_adv.upper(): - rhsa_data = requests.get( - f"https://access.redhat.com/hydra/rest/securitydata/cvrf/{rh_adv}.json" - ).json() # nopep8 - value = rhsa_data["cvrfdoc"]["aggregate_severity"] - rhsa_aggregate_severity = VulnerabilitySeverity( - system=scoring_systems["rhas"], - value=value, - ) + for rh_adv in advisory_data["advisories"]: + # RH provides 3 types of advisories RHSA, RHBA, RHEA. Only RHSA's contain severity score. + # See https://access.redhat.com/articles/2130961 for more details. + + if "RHSA" in rh_adv.upper(): + rhsa_data = requests_session.get( + f"https://access.redhat.com/hydra/rest/securitydata/cvrf/{rh_adv}.json" + ).json() # nopep8 + value = rhsa_data["cvrfdoc"]["aggregate_severity"] + rhsa_aggregate_severity = VulnerabilitySeverity( + system=scoring_systems["rhas"], + value=value, + ) - references.append( - Reference( - severities=[rhsa_aggregate_severity], - url="https://access.redhat.com/errata/{}".format(rh_adv), - reference_id=rh_adv, + references.append( + Reference( + severities=[rhsa_aggregate_severity], + url="https://access.redhat.com/errata/{}".format(rh_adv), + reference_id=rh_adv, + ) ) - ) - else: - references.append(Reference(severities=[], url=url, reference_id=rh_adv)) + else: + references.append( + Reference(severities=[], url=url, reference_id=rh_adv) + ) - redhat_scores = [] - cvssv3_score = advisory_data.get("cvss3_score") - if cvssv3_score: - redhat_scores.append( - VulnerabilitySeverity( - system=scoring_systems["cvssv3"], - value=cvssv3_score, + redhat_scores = [] + cvssv3_score = advisory_data.get("cvss3_score") + if cvssv3_score: + redhat_scores.append( + VulnerabilitySeverity( + system=scoring_systems["cvssv3"], + value=cvssv3_score, + ) ) - ) - cvssv3_vector = advisory_data.get("cvss3_scoring_vector") - if cvssv3_vector: - redhat_scores.append( - VulnerabilitySeverity( - system=scoring_systems["cvssv3_vector"], - value=cvssv3_vector, + cvssv3_vector = advisory_data.get("cvss3_scoring_vector") + if cvssv3_vector: + redhat_scores.append( + VulnerabilitySeverity( + system=scoring_systems["cvssv3_vector"], + value=cvssv3_vector, + ) ) - ) - references.append(Reference(severities=redhat_scores, url=advisory_data["resource_url"])) - return Advisory( - vulnerability_id=advisory_data["CVE"], - summary=advisory_data["bugzilla_description"], - impacted_package_urls=affected_purls, - references=references, - ) + references.append( + Reference(severities=redhat_scores, url=advisory_data["resource_url"]) + ) + return Advisory( + vulnerability_id=advisory_data["CVE"], + summary=advisory_data["bugzilla_description"], + impacted_package_urls=affected_purls, + references=references, + ) + except Exception as e: + print(e) + pdb.set_trace() def rpm_to_purl(rpm_string): From aff0065ed4b7d3af2c5fbc1e575eab28327ef235 Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Thu, 1 Apr 2021 00:53:53 +0530 Subject: [PATCH 22/28] Not all RHSA errata have a CVRF document This is mentioned in the NOTE of "2.1 List all CVRFs" of https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0/html/red_hat_security_data_api/cvrf Such a case would lead to a crash before this commit. Eg: https://access.redhat.com/hydra/rest/securitydata/cvrf/RHSA-2005:835.json No cvrfdoc would be found in the statement value = rhsa_data["cvrfdoc"]["aggregate_severity"] Signed-off-by: Hritik Vijay --- vulnerabilities/importers/redhat.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/vulnerabilities/importers/redhat.py b/vulnerabilities/importers/redhat.py index a591d3e72..0fbc10b15 100644 --- a/vulnerabilities/importers/redhat.py +++ b/vulnerabilities/importers/redhat.py @@ -127,15 +127,19 @@ def to_advisory(advisory_data): rhsa_data = requests_session.get( f"https://access.redhat.com/hydra/rest/securitydata/cvrf/{rh_adv}.json" ).json() # nopep8 - value = rhsa_data["cvrfdoc"]["aggregate_severity"] - rhsa_aggregate_severity = VulnerabilitySeverity( - system=scoring_systems["rhas"], - value=value, - ) + + rhsa_aggregate_severities = [] + if rhsa_data.get("cvrfdoc"): + # not all RHSA errata have a corresponding CVRF document + value = rhsa_data["cvrfdoc"]["aggregate_severity"] + rhsa_aggregate_severities.append(VulnerabilitySeverity( + system=scoring_systems["rhas"], + value=value, + )) references.append( Reference( - severities=[rhsa_aggregate_severity], + severities=rhsa_aggregate_severities, url="https://access.redhat.com/errata/{}".format(rh_adv), reference_id=rh_adv, ) From b0329fae15bbb0504d90d12bd99bd2a701504443 Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Thu, 1 Apr 2021 16:13:31 +0530 Subject: [PATCH 23/28] Remove debugging symbols and black -l 100 This finally fixes https://github.com/nexB/vulnerablecode/issues/398 Signed-off-by: Hritik Vijay --- vulnerabilities/importers/redhat.py | 160 +++++++++++++--------------- 1 file changed, 76 insertions(+), 84 deletions(-) diff --git a/vulnerabilities/importers/redhat.py b/vulnerabilities/importers/redhat.py index 0fbc10b15..9415cd94a 100644 --- a/vulnerabilities/importers/redhat.py +++ b/vulnerabilities/importers/redhat.py @@ -32,8 +32,6 @@ from vulnerabilities.severity_systems import scoring_systems -import pdb - class RedhatDataSource(DataSource): CONFIG_CLASS = DataSourceConfiguration @@ -85,102 +83,96 @@ def fetch(): def to_advisory(advisory_data): - try: - affected_purls = [] - if advisory_data.get("affected_packages"): - for rpm in advisory_data["affected_packages"]: - purl = rpm_to_purl(rpm) - if purl: - affected_purls.append(purl) - - references = [] - bugzilla = advisory_data.get("bugzilla") - if bugzilla: - url = "https://bugzilla.redhat.com/show_bug.cgi?id={}".format(bugzilla) - bugzilla_data = requests_session.get( - f"https://bugzilla.redhat.com/rest/bug/{bugzilla}" - ).json() - if ( - bugzilla_data.get("bugs") - and len(bugzilla_data["bugs"]) - and bugzilla_data["bugs"][0].get("severity") - ): - bugzilla_severity_val = bugzilla_data["bugs"][0]["severity"] - bugzilla_severity = VulnerabilitySeverity( - system=scoring_systems["rhbs"], - value=bugzilla_severity_val, - ) + affected_purls = [] + if advisory_data.get("affected_packages"): + for rpm in advisory_data["affected_packages"]: + purl = rpm_to_purl(rpm) + if purl: + affected_purls.append(purl) + + references = [] + bugzilla = advisory_data.get("bugzilla") + if bugzilla: + url = "https://bugzilla.redhat.com/show_bug.cgi?id={}".format(bugzilla) + bugzilla_data = requests_session.get( + f"https://bugzilla.redhat.com/rest/bug/{bugzilla}" + ).json() + if ( + bugzilla_data.get("bugs") + and len(bugzilla_data["bugs"]) + and bugzilla_data["bugs"][0].get("severity") + ): + bugzilla_severity_val = bugzilla_data["bugs"][0]["severity"] + bugzilla_severity = VulnerabilitySeverity( + system=scoring_systems["rhbs"], + value=bugzilla_severity_val, + ) - references.append( - Reference( - severities=[bugzilla_severity], - url=url, - reference_id=bugzilla, - ) + references.append( + Reference( + severities=[bugzilla_severity], + url=url, + reference_id=bugzilla, ) + ) - for rh_adv in advisory_data["advisories"]: - # RH provides 3 types of advisories RHSA, RHBA, RHEA. Only RHSA's contain severity score. - # See https://access.redhat.com/articles/2130961 for more details. - - if "RHSA" in rh_adv.upper(): - rhsa_data = requests_session.get( - f"https://access.redhat.com/hydra/rest/securitydata/cvrf/{rh_adv}.json" - ).json() # nopep8 - - rhsa_aggregate_severities = [] - if rhsa_data.get("cvrfdoc"): - # not all RHSA errata have a corresponding CVRF document - value = rhsa_data["cvrfdoc"]["aggregate_severity"] - rhsa_aggregate_severities.append(VulnerabilitySeverity( + for rh_adv in advisory_data["advisories"]: + # RH provides 3 types of advisories RHSA, RHBA, RHEA. Only RHSA's contain severity score. + # See https://access.redhat.com/articles/2130961 for more details. + + if "RHSA" in rh_adv.upper(): + rhsa_data = requests_session.get( + f"https://access.redhat.com/hydra/rest/securitydata/cvrf/{rh_adv}.json" + ).json() # nopep8 + + rhsa_aggregate_severities = [] + if rhsa_data.get("cvrfdoc"): + # not all RHSA errata have a corresponding CVRF document + value = rhsa_data["cvrfdoc"]["aggregate_severity"] + rhsa_aggregate_severities.append( + VulnerabilitySeverity( system=scoring_systems["rhas"], value=value, - )) - - references.append( - Reference( - severities=rhsa_aggregate_severities, - url="https://access.redhat.com/errata/{}".format(rh_adv), - reference_id=rh_adv, ) ) - else: - references.append( - Reference(severities=[], url=url, reference_id=rh_adv) - ) - - redhat_scores = [] - cvssv3_score = advisory_data.get("cvss3_score") - if cvssv3_score: - redhat_scores.append( - VulnerabilitySeverity( - system=scoring_systems["cvssv3"], - value=cvssv3_score, + references.append( + Reference( + severities=rhsa_aggregate_severities, + url="https://access.redhat.com/errata/{}".format(rh_adv), + reference_id=rh_adv, ) ) - cvssv3_vector = advisory_data.get("cvss3_scoring_vector") - if cvssv3_vector: - redhat_scores.append( - VulnerabilitySeverity( - system=scoring_systems["cvssv3_vector"], - value=cvssv3_vector, - ) - ) + else: + references.append(Reference(severities=[], url=url, reference_id=rh_adv)) - references.append( - Reference(severities=redhat_scores, url=advisory_data["resource_url"]) + redhat_scores = [] + cvssv3_score = advisory_data.get("cvss3_score") + if cvssv3_score: + redhat_scores.append( + VulnerabilitySeverity( + system=scoring_systems["cvssv3"], + value=cvssv3_score, + ) ) - return Advisory( - vulnerability_id=advisory_data["CVE"], - summary=advisory_data["bugzilla_description"], - impacted_package_urls=affected_purls, - references=references, + + cvssv3_vector = advisory_data.get("cvss3_scoring_vector") + if cvssv3_vector: + redhat_scores.append( + VulnerabilitySeverity( + system=scoring_systems["cvssv3_vector"], + value=cvssv3_vector, + ) ) - except Exception as e: - print(e) - pdb.set_trace() + + references.append(Reference(severities=redhat_scores, url=advisory_data["resource_url"])) + return Advisory( + vulnerability_id=advisory_data["CVE"], + summary=advisory_data["bugzilla_description"], + impacted_package_urls=affected_purls, + references=references, + ) def rpm_to_purl(rpm_string): From c90a8717b79591e7fe788c483bf353225cf01e6d Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Fri, 2 Apr 2021 00:52:21 +0530 Subject: [PATCH 24/28] Mock requests_session in redhat importer Previous commits replace the usage of requests.get() altogether with a custom requests_session which provides better 5xx error handling. It is now required to mock that object in this test. IMHO it would make more sense to update this test altogether to use the real endpoints against some real data. Signed-off-by: Hritik Vijay --- vulnerabilities/tests/test_redhat_importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnerabilities/tests/test_redhat_importer.py b/vulnerabilities/tests/test_redhat_importer.py index ba7631399..967b01684 100644 --- a/vulnerabilities/tests/test_redhat_importer.py +++ b/vulnerabilities/tests/test_redhat_importer.py @@ -139,7 +139,7 @@ def test_to_advisory(self): } for adv in data: with unittest.mock.patch( - "vulnerabilities.importers.redhat.requests.get", return_value=mock_resp + "vulnerabilities.importers.redhat.requests_session.get", return_value=mock_resp ): adv = redhat.to_advisory(adv) found_advisories.append(adv) From dbb88785a3afa478e75737f247eb59c255622672 Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Tue, 30 Mar 2021 22:57:08 +0530 Subject: [PATCH 25/28] Add unspecified scoring system Signed-off-by: Hritik Vijay --- vulnerabilities/severity_systems.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/vulnerabilities/severity_systems.py b/vulnerabilities/severity_systems.py index e2357856e..794a58227 100644 --- a/vulnerabilities/severity_systems.py +++ b/vulnerabilities/severity_systems.py @@ -81,4 +81,10 @@ def as_score(self, value): url="https://www.first.org/cvss/specification-document#Qualitative-Severity-Rating-Scale", notes="A textual interpretation of severity. Has values like HIGH, MODERATE etc", ), + "unspecified": ScoringSystem( + identifier="unspecified", + name="unspecified", + url="https://example.com", + notes="Severity with unspecified scoring system. Contains values like High, Low etc", + ), } From cdaf81ef79916a4aa7cbc0566fd6365ab32c9852 Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Fri, 2 Apr 2021 14:44:13 +0530 Subject: [PATCH 26/28] Rename unspecified to generic system Signed-off-by: Hritik Vijay --- vulnerabilities/severity_systems.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/vulnerabilities/severity_systems.py b/vulnerabilities/severity_systems.py index 794a58227..33594268a 100644 --- a/vulnerabilities/severity_systems.py +++ b/vulnerabilities/severity_systems.py @@ -81,10 +81,10 @@ def as_score(self, value): url="https://www.first.org/cvss/specification-document#Qualitative-Severity-Rating-Scale", notes="A textual interpretation of severity. Has values like HIGH, MODERATE etc", ), - "unspecified": ScoringSystem( - identifier="unspecified", - name="unspecified", - url="https://example.com", - notes="Severity with unspecified scoring system. Contains values like High, Low etc", + "generic_textual": ScoringSystem( + identifier="generic_textual", + name="Generic textual severity rating", + url="", + notes="Severity for unknown scoring systems. Contains generic textual values like High, Low etc", ), } From e0bd0f0fc835c1475cf764d6be3c72b63d719c49 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Sat, 13 Mar 2021 12:00:26 +0530 Subject: [PATCH 27/28] Add tests for checking upstream data sources Signed-off-by: Shivam Sandbhor --- .github/workflows/main.yml | 2 +- .github/workflows/upstream_test.yml | 51 ++++++++++++++++++++++++++ vulnerabilities/tests/test_upstream.py | 18 +++++++++ 3 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/upstream_test.yml create mode 100644 vulnerabilities/tests/test_upstream.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 01418af19..d05b5f0ba 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -38,7 +38,7 @@ jobs: pip install -r requirements.txt - name: Run tests - run: python -m pytest + run: python -m pytest -v -m "not webtest" env: # The hostname, username used to communicate with the PostgreSQL service container POSTGRES_HOST: localhost diff --git a/.github/workflows/upstream_test.yml b/.github/workflows/upstream_test.yml new file mode 100644 index 000000000..30cbcdd90 --- /dev/null +++ b/.github/workflows/upstream_test.yml @@ -0,0 +1,51 @@ +on: + workflow_dispatch: # allow manual execution + push: + schedule: + # run on every 9 o'clock + - cron: '0 9 * * *' + +jobs: + unit_tests: + runs-on: ubuntu-latest + + services: + # Label used to access the service container + postgres: + image: postgres + env: + POSTGRES_PASSWORD: vulnerablecode + POSTGRES_DB: vulnerablecode + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + # Maps tcp port 5432 on service container to the host + - 5432:5432 + steps: + - name: Check out repository code + uses: actions/checkout@v2 + + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Install dependencies + run: | + sudo apt install python3-dev postgresql libpq-dev build-essential libxml2-dev libxslt1-dev + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Run tests + run: pytest -v -m webtest + env: + # The hostname, username used to communicate with the PostgreSQL service container + POSTGRES_HOST: localhost + VC_DB_USER: postgres + POSTGRES_PORT: 5432 + DJANGO_DEV: 1 + GH_TOKEN: 1 \ No newline at end of file diff --git a/vulnerabilities/tests/test_upstream.py b/vulnerabilities/tests/test_upstream.py new file mode 100644 index 000000000..a875f570e --- /dev/null +++ b/vulnerabilities/tests/test_upstream.py @@ -0,0 +1,18 @@ +import pytest +from vulnerabilities import importers +from vulnerabilities.importer_yielder import IMPORTER_REGISTRY + + +@pytest.mark.webtest +@pytest.mark.parametrize( + ("data_source", "config"), + ((data["data_source"], data["data_source_cfg"]) for data in IMPORTER_REGISTRY), +) +def test_updated_advisories(data_source, config): + + if not data_source == "GitHubAPIDataSource": + data_src = getattr(importers, data_source) + data_src = data_src(batch_size=1, config=config) + with data_src: + for i in data_src.updated_advisories(): + pass From 60370ee020bddef824d2c4aad63005492a6a5321 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Fri, 19 Mar 2021 14:49:46 +0530 Subject: [PATCH 28/28] Update travis config Signed-off-by: Shivam Sandbhor --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e0f5a631a..e47738ee8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ before_script: script: - ./manage.py collectstatic - - python -m pytest + - python -m pytest -v -m "not webtest" notifications: email: false