From 8e4c30d396353948cd4b3f222a5c5212e6ff7587 Mon Sep 17 00:00:00 2001 From: AmitGupta7580 Date: Thu, 25 Mar 2021 19:11:20 +0530 Subject: [PATCH 1/5] Added documentation of /api/package/bulk_search/ API Signed-off-by: AmitGupta7580 --- vulnerabilities/api.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/vulnerabilities/api.py b/vulnerabilities/api.py index 07efcfc55..0b5e28357 100644 --- a/vulnerabilities/api.py +++ b/vulnerabilities/api.py @@ -136,13 +136,19 @@ class PackageViewSet(viewsets.ReadOnlyModelViewSet): filter_backends = (filters.DjangoFilterBackend,) filterset_class = PackageFilterSet - # TODO: Fix the swagger documentation for this endpoint - @extend_schema(request=placeholder_serializer, responses=placeholder_serializer) + class PackageBulkSearchRequestSerializer(serializers.Serializer): + purls = serializers.ListField( + child=serializers.CharField(max_length=100) + ) + + class PackageBulkSearchResponseSerializer(serializers.Serializer): + result = serializers.ListField( + child=PackageSerializer() + ) + + @extend_schema(request=PackageBulkSearchRequestSerializer, responses=PackageBulkSearchResponseSerializer) @action(detail=False, methods=["post"]) def bulk_search(self, request): - """ - See https://github.com/nexB/vulnerablecode/pull/369#issuecomment-796877606 for docs - """ response = [] purls = request.data.get("purls", []) or [] if not purls or not isinstance(purls, list): @@ -166,8 +172,8 @@ def bulk_search(self, request): purl_response["unresolved_vulnerabilities"] = [] purl_response["resolved_vulnerabilities"] = [] response.append(purl_response) - - return Response(response) + res = {'result': response} + return Response(res) class VulnerabilityFilterSet(filters.FilterSet): From 2ff0749e97392ab6f797ff5972925a950e342e51 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Mon, 15 Mar 2021 13:50:13 +0530 Subject: [PATCH 2/5] Fix various importer errors - Update debian importer's schema validation - Add tests for msr2019 importer. Signed-off-by: Shivam Sandbhor Signed-off-by: AmitGupta7580 --- vulnerabilities/importers/debian.py | 1 + .../importers/project_kb_msr2019.py | 4 +- .../test_data/kbmsr2019/test_msr_data.csv | 7 + vulnerabilities/tests/test_msr2019.py | 141 ++++++++++++++++++ vulnerabilities/tests/test_nginx.py | 1 - 5 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 vulnerabilities/tests/test_data/kbmsr2019/test_msr_data.csv create mode 100644 vulnerabilities/tests/test_msr2019.py diff --git a/vulnerabilities/importers/debian.py b/vulnerabilities/importers/debian.py index ae9004f37..28d499980 100644 --- a/vulnerabilities/importers/debian.py +++ b/vulnerabilities/importers/debian.py @@ -45,6 +45,7 @@ def validate_schema(advisory_dict): deb_versions = [ "bullseye", + "bullseye-security", "buster", "buster-security", "sid", diff --git a/vulnerabilities/importers/project_kb_msr2019.py b/vulnerabilities/importers/project_kb_msr2019.py index 156cfb82f..b9f797c48 100644 --- a/vulnerabilities/importers/project_kb_msr2019.py +++ b/vulnerabilities/importers/project_kb_msr2019.py @@ -47,7 +47,7 @@ class ProjectKBMSRDataSource(DataSource): CONFIG_CLASS = ProjectKBDataSourceConfiguration - url = "https://raw.githubusercontent.com/SAP/project-kb/master/MSR2019/dataset/vulas_db_msr2019_release.csv" # nopep8 + url = "https://raw.githubusercontent.com/SAP/project-kb/master/MSR2019/dataset/vulas_db_msr2019_release.csv" def updated_advisories(self): if create_etag(data_src=self, url=self.url, etag_key="ETag"): @@ -82,7 +82,7 @@ def to_advisories(csv_reader): summary="", impacted_package_urls=[], references=[reference], - cve_id=vuln_id, + vulnerability_id=vuln_id, ) ) diff --git a/vulnerabilities/tests/test_data/kbmsr2019/test_msr_data.csv b/vulnerabilities/tests/test_data/kbmsr2019/test_msr_data.csv new file mode 100644 index 000000000..f0a35d932 --- /dev/null +++ b/vulnerabilities/tests/test_data/kbmsr2019/test_msr_data.csv @@ -0,0 +1,7 @@ +CVE-2018-11040,https://github.com/spring-projects/spring-framework,874859493bbda59739c38c7e52eb3625f247b93,pos +CVE-2013-6408,https://github.com/apache/lucene-solr,7239a57a51ea0f4d05dd330ce5e15e4f72f72747,pos +CVE-2015-6748,https://github.com/jhy/jsoup,4edb78991f8d0bf87dafde5e01ccd8922065c9b2,pos +CVE-2018-14658,https://github.com/keycloak/keycloak,a957e118e6efb35fe7ef3a62acd66341a6523cb7,pos +CVE-2017-1000355,https://github.com/jenkinsci/jenkins,701ea95a52afe53bee28f76a3f96eb0e578852e9,pos +CVE-2018-1000844,https://github.com/square/retrofit,97057aaae42e54bfbee8acfa8af7dcf37e812342,pos +HTTPCLIENT-1803,https://github.com/apache/httpcomponents-client,0554271750599756d4946c0d7ba43d04b1a7b22,pos \ No newline at end of file diff --git a/vulnerabilities/tests/test_msr2019.py b/vulnerabilities/tests/test_msr2019.py new file mode 100644 index 000000000..df8956f3b --- /dev/null +++ b/vulnerabilities/tests/test_msr2019.py @@ -0,0 +1,141 @@ +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software tool from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import csv +import os +from unittest import TestCase +from unittest.mock import patch + +from packageurl import PackageURL + +from vulnerabilities.data_source import Advisory +from vulnerabilities.data_source import Reference +from vulnerabilities.importers import ProjectKBMSRDataSource + + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, "test_data/kbmsr2019", "test_msr_data.csv") + + +class TestProjectKBMSRDataSource(TestCase): + def test_to_advisories(self): + with open(TEST_DATA) as f: + lines = [l for l in f.readlines()] + test_data = csv.reader(lines) + + found_advisories = ProjectKBMSRDataSource.to_advisories(test_data) + found_advisories = list(map(Advisory.normalized, found_advisories)) + expected_advisories = [ + Advisory( + summary="", + vulnerability_id="CVE-2018-11040", + impacted_package_urls=set(), + resolved_package_urls=set(), + references=[ + Reference( + reference_id="", + url="https://github.com/spring-projects/spring-framework/commit/874859493bbda59739c38c7e52eb3625f247b93", + severities=[], + ) + ], + ), + Advisory( + summary="", + vulnerability_id="CVE-2013-6408", + impacted_package_urls=set(), + resolved_package_urls=set(), + references=[ + Reference( + reference_id="", + url="https://github.com/apache/lucene-solr/commit/7239a57a51ea0f4d05dd330ce5e15e4f72f72747", + severities=[], + ) + ], + ), + Advisory( + summary="", + vulnerability_id="CVE-2015-6748", + impacted_package_urls=set(), + resolved_package_urls=set(), + references=[ + Reference( + reference_id="", + url="https://github.com/jhy/jsoup/commit/4edb78991f8d0bf87dafde5e01ccd8922065c9b2", + severities=[], + ) + ], + ), + Advisory( + summary="", + vulnerability_id="CVE-2018-14658", + impacted_package_urls=set(), + resolved_package_urls=set(), + references=[ + Reference( + reference_id="", + url="https://github.com/keycloak/keycloak/commit/a957e118e6efb35fe7ef3a62acd66341a6523cb7", + severities=[], + ) + ], + ), + Advisory( + summary="", + vulnerability_id="CVE-2017-1000355", + impacted_package_urls=set(), + resolved_package_urls=set(), + references=[ + Reference( + reference_id="", + url="https://github.com/jenkinsci/jenkins/commit/701ea95a52afe53bee28f76a3f96eb0e578852e9", + severities=[], + ) + ], + ), + Advisory( + summary="", + vulnerability_id="CVE-2018-1000844", + impacted_package_urls=set(), + resolved_package_urls=set(), + references=[ + Reference( + reference_id="", + url="https://github.com/square/retrofit/commit/97057aaae42e54bfbee8acfa8af7dcf37e812342", + severities=[], + ) + ], + ), + Advisory( + summary="", + vulnerability_id="", + impacted_package_urls=set(), + resolved_package_urls=set(), + references=[ + Reference( + reference_id="HTTPCLIENT-1803", + url="https://github.com/apache/httpcomponents-client/commit/0554271750599756d4946c0d7ba43d04b1a7b22", + severities=[], + ) + ], + ), + ] + + assert expected_advisories == found_advisories diff --git a/vulnerabilities/tests/test_nginx.py b/vulnerabilities/tests/test_nginx.py index a7f0f3837..1281bc67e 100644 --- a/vulnerabilities/tests/test_nginx.py +++ b/vulnerabilities/tests/test_nginx.py @@ -31,7 +31,6 @@ from vulnerabilities.importers.nginx import NginxDataSource from vulnerabilities.package_managers import GitHubTagsAPI -# BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data/nginx", "security_advisories.html") From de5301ac9fee27da957695ad8adc4f9076017104 Mon Sep 17 00:00:00 2001 From: Amit_Gupta Date: Thu, 25 Mar 2021 23:55:27 +0530 Subject: [PATCH 3/5] Update test_api.py Signed-off-by: AmitGupta7580 --- vulnerabilities/tests/test_api.py | 100 +++++++++++++++--------------- 1 file changed, 51 insertions(+), 49 deletions(-) diff --git a/vulnerabilities/tests/test_api.py b/vulnerabilities/tests/test_api.py index f95024e31..ebf3bfe6b 100644 --- a/vulnerabilities/tests/test_api.py +++ b/vulnerabilities/tests/test_api.py @@ -256,55 +256,57 @@ def test_bulk_packages_api(self): content_type="application/json", ).json() - expected_response = [ - { - "name": "doesnotexist", - "namespace": "debian", - "qualifiers": {"distro": "jessie"}, - "resolved_vulnerabilities": [], - "subpath": None, - "type": "deb", - "unresolved_vulnerabilities": [], - "version": "0.9.7-10", - }, - { - "name": "datadog-api-client", - "namespace": "com.datadoghq", - "purl": "pkg:maven/com.datadoghq/datadog-api-client@1.0.0-beta.7", - "qualifiers": {}, - "resolved_vulnerabilities": [], - "subpath": "", - "type": "maven", - "unresolved_vulnerabilities": [ - { - "references": [ - { - "reference_id": "", - "scores": [], - "source": "", - "url": "https://nvd.nist.gov/vuln/detail/CVE-2021-21331", - }, - { - "reference_id": "GHSA-2cxf-6567-7pp6", - "scores": [{"scoring_system": "cvssv3.1_qr", "value": "LOW"}], - "source": "", - "url": "https://github.com/DataDog/datadog-api-client-java/security/advisories/GHSA-2cxf-6567-7pp6", - }, - { - "reference_id": "GHSA-2cxf-6567-7pp6", - "scores": [], - "source": "", - "url": "https://github.com/advisories/GHSA-2cxf-6567-7pp6", - }, - ], - "url": "http://testserver/api/vulnerabilities/60", - "vulnerability_id": "CVE-2021-21331", - } - ], - "url": "http://testserver/api/packages/3467", - "version": "1.0.0-beta.7", - }, - ] + expected_response = { + "result": [ + { + "name": "doesnotexist", + "namespace": "debian", + "qualifiers": {"distro": "jessie"}, + "resolved_vulnerabilities": [], + "subpath": None, + "type": "deb", + "unresolved_vulnerabilities": [], + "version": "0.9.7-10", + }, + { + "name": "datadog-api-client", + "namespace": "com.datadoghq", + "purl": "pkg:maven/com.datadoghq/datadog-api-client@1.0.0-beta.7", + "qualifiers": {}, + "resolved_vulnerabilities": [], + "subpath": "", + "type": "maven", + "unresolved_vulnerabilities": [ + { + "references": [ + { + "reference_id": "", + "scores": [], + "source": "", + "url": "https://nvd.nist.gov/vuln/detail/CVE-2021-21331", + }, + { + "reference_id": "GHSA-2cxf-6567-7pp6", + "scores": [{"scoring_system": "cvssv3.1_qr", "value": "LOW"}], + "source": "", + "url": "https://github.com/DataDog/datadog-api-client-java/security/advisories/GHSA-2cxf-6567-7pp6", + }, + { + "reference_id": "GHSA-2cxf-6567-7pp6", + "scores": [], + "source": "", + "url": "https://github.com/advisories/GHSA-2cxf-6567-7pp6", + }, + ], + "url": "http://testserver/api/vulnerabilities/60", + "vulnerability_id": "CVE-2021-21331", + } + ], + "url": "http://testserver/api/packages/3467", + "version": "1.0.0-beta.7", + }, + ] + } assert cleaned_response(expected_response) == cleaned_response(response) def test_invalid_request_bulk_packages(self): From f06b7d892938c94cb99a13862c4361c5236e9a4b Mon Sep 17 00:00:00 2001 From: Amit_Gupta Date: Fri, 26 Mar 2021 01:19:46 +0530 Subject: [PATCH 4/5] Update test_api.py Signed-off-by: AmitGupta7580 --- vulnerabilities/tests/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vulnerabilities/tests/test_api.py b/vulnerabilities/tests/test_api.py index ebf3bfe6b..130f01d49 100644 --- a/vulnerabilities/tests/test_api.py +++ b/vulnerabilities/tests/test_api.py @@ -46,7 +46,7 @@ def cleaned_response(response): - sort lists with a stable order """ cleaned_response = [] - response_copy = sorted(response, key=lambda x: x.get("purl", "")) + response_copy = sorted(response["result"], key=lambda x: x.get("purl", "")) for package_data in response_copy: package_data["unresolved_vulnerabilities"] = sorted( package_data["unresolved_vulnerabilities"], key=lambda x: x["vulnerability_id"] From 79ee35a061653be31ba1135a7d738a3352e40802 Mon Sep 17 00:00:00 2001 From: Amit_Gupta Date: Fri, 26 Mar 2021 02:53:11 +0530 Subject: [PATCH 5/5] Update api.py Signed-off-by: AmitGupta7580 --- vulnerabilities/api.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/vulnerabilities/api.py b/vulnerabilities/api.py index 0b5e28357..0441e8aef 100644 --- a/vulnerabilities/api.py +++ b/vulnerabilities/api.py @@ -136,17 +136,13 @@ class PackageViewSet(viewsets.ReadOnlyModelViewSet): filter_backends = (filters.DjangoFilterBackend,) filterset_class = PackageFilterSet - class PackageBulkSearchRequestSerializer(serializers.Serializer): - purls = serializers.ListField( - child=serializers.CharField(max_length=100) - ) + class PackageBulkRequestSerializer(serializers.Serializer): + purls = serializers.ListField(child=serializers.CharField(max_length=100)) - class PackageBulkSearchResponseSerializer(serializers.Serializer): - result = serializers.ListField( - child=PackageSerializer() - ) + class PackageBulkResponseSerializer(serializers.Serializer): + result = serializers.ListField(child=PackageSerializer()) - @extend_schema(request=PackageBulkSearchRequestSerializer, responses=PackageBulkSearchResponseSerializer) + @extend_schema(request=PackageBulkRequestSerializer, responses=PackageBulkResponseSerializer) @action(detail=False, methods=["post"]) def bulk_search(self, request): response = [] @@ -172,7 +168,7 @@ def bulk_search(self, request): purl_response["unresolved_vulnerabilities"] = [] purl_response["resolved_vulnerabilities"] = [] response.append(purl_response) - res = {'result': response} + res = {"result": response} return Response(res)