From e9172cb67341f0d45f97bce3f65e06fbe3baa0c4 Mon Sep 17 00:00:00 2001 From: AmitGupta7580 Date: Sun, 4 Apr 2021 17:21:16 +0530 Subject: [PATCH 1/9] Update Apache httpd importer from importing XML to JSON data Signed-off-by: AmitGupta7580 --- vulnerabilities/importers/apache_httpd.py | 84 +++++++++++++---------- 1 file changed, 48 insertions(+), 36 deletions(-) diff --git a/vulnerabilities/importers/apache_httpd.py b/vulnerabilities/importers/apache_httpd.py index 41a9899d6..accae75f7 100644 --- a/vulnerabilities/importers/apache_httpd.py +++ b/vulnerabilities/importers/apache_httpd.py @@ -21,14 +21,18 @@ # Visit https://github.com/nexB/vulnerablecode/ for support and download. import dataclasses -from xml.etree import ElementTree +from bs4 import BeautifulSoup import requests +import re from packageurl import PackageURL from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource from vulnerabilities.data_source import DataSourceConfiguration +from vulnerabilities.data_source import Reference +from vulnerabilities.data_source import VulnerabilitySeverity +from vulnerabilities.severity_systems import scoring_systems from vulnerabilities.helpers import create_etag @@ -40,7 +44,7 @@ class ApacheHTTPDDataSourceConfiguration(DataSourceConfiguration): class ApacheHTTPDDataSource(DataSource): CONFIG_CLASS = ApacheHTTPDDataSourceConfiguration - url = "https://httpd.apache.org/security/vulnerabilities-httpd.xml" + url = "https://httpd.apache.org/security/json/" def updated_advisories(self): # Etags are like hashes of web responses. We maintain @@ -49,47 +53,55 @@ def updated_advisories(self): # skips processing the response further to avoid duplicate work if create_etag(data_src=self, url=self.url, etag_key="ETag"): - data = fetch_xml(self.url) - advisories = to_advisories(data) + links = fetch_links(self.url) + advisories = [] + for link in links: + data = requests.get(link).json() + advisories.append(self.to_advisories(data)) return self.batch_advisories(advisories) return [] + def to_advisories(self, data): + cve = data["CVE_data_meta"]["ID"] + description = data["description"]["description_data"] + summary = next((item["value"] for item in description if item["lang"] == "eng"), "") + reference = Reference(reference_id=cve, url=self.url + cve + ".json") -def to_advisories(data): - advisories = [] - for issue in data: resolved_packages = [] impacted_packages = [] - for info in issue: - if info.tag == "cve": - cve = info.attrib["name"] - - if info.tag == "title": - summary = info.text - - if info.tag == "fixed": - resolved_packages.append( - PackageURL(type="apache", name="httpd", version=info.attrib["version"]) - ) - - if info.tag == "affects" or info.tag == "maybeaffects": - impacted_packages.append( - PackageURL(type="apache", name="httpd", version=info.attrib["version"]) - ) - - advisories.append( - Advisory( - vulnerability_id=cve, - summary=summary, - impacted_package_urls=impacted_packages, - resolved_package_urls=resolved_packages, - ) - ) - return advisories + for vendor in data["affects"]["vendor"]["vendor_data"]: + for products in vendor["product"]["product_data"]: + for version in products["version"]["version_data"]: + version_value = version["version_value"] + + if version["version_affected"] == "<": + resolved_packages.append( + PackageURL(type="apache", name="httpd", version=version_value) + ) + + else: + impacted_packages.append( + PackageURL(type="apache", name="httpd", version=version_value) + ) + + return Advisory( + vulnerability_id=cve, + summary=summary, + impacted_package_urls=impacted_packages, + resolved_package_urls=resolved_packages, + references=[reference], + ) -def fetch_xml(url): - resp = requests.get(url).content - return ElementTree.fromstring(resp) +def fetch_links(url): + links = [] + data = requests.get(url).content + soup = BeautifulSoup(data, features="lxml") + for tag in soup.find_all("a"): + link = tag.get("href") + if not re.search("^.*json$", link): + continue + links.append(url + link) + return links From 4bd6a3849f3c8c884a0781cb3841eab9a7525a66 Mon Sep 17 00:00:00 2001 From: AmitGupta7580 Date: Tue, 6 Apr 2021 01:26:22 +0530 Subject: [PATCH 2/9] Add severity system and test file for apache_httpd Signed-off-by: AmitGupta7580 --- vulnerabilities/importers/apache_httpd.py | 13 ++- vulnerabilities/tests/test_apache_httpd.py | 96 +++++++++++++++++++ .../test_data/apache_httpd/CVE-1999-1199.json | 86 +++++++++++++++++ 3 files changed, 190 insertions(+), 5 deletions(-) create mode 100644 vulnerabilities/tests/test_apache_httpd.py create mode 100644 vulnerabilities/tests/test_data/apache_httpd/CVE-1999-1199.json diff --git a/vulnerabilities/importers/apache_httpd.py b/vulnerabilities/importers/apache_httpd.py index accae75f7..c5da0648f 100644 --- a/vulnerabilities/importers/apache_httpd.py +++ b/vulnerabilities/importers/apache_httpd.py @@ -57,17 +57,20 @@ def updated_advisories(self): advisories = [] for link in links: data = requests.get(link).json() - advisories.append(self.to_advisories(data)) + advisories.append(self.to_advisory(data)) return self.batch_advisories(advisories) return [] - def to_advisories(self, data): + def to_advisory(self, data): cve = data["CVE_data_meta"]["ID"] description = data["description"]["description_data"] summary = next((item["value"] for item in description if item["lang"] == "eng"), "") - reference = Reference(reference_id=cve, url=self.url + cve + ".json") - + severity = VulnerabilitySeverity( + system=scoring_systems["generic_textual"], + value=data["impact"][0]["other"], + ) + reference = Reference(reference_id=cve, url=self.url + cve + ".json", severities=[severity]) resolved_packages = [] impacted_packages = [] @@ -101,7 +104,7 @@ def fetch_links(url): soup = BeautifulSoup(data, features="lxml") for tag in soup.find_all("a"): link = tag.get("href") - if not re.search("^.*json$", link): + if not re.search("^CVE.*json$", link): continue links.append(url + link) return links diff --git a/vulnerabilities/tests/test_apache_httpd.py b/vulnerabilities/tests/test_apache_httpd.py new file mode 100644 index 000000000..3f28c59be --- /dev/null +++ b/vulnerabilities/tests/test_apache_httpd.py @@ -0,0 +1,96 @@ +# Copyright (c) nexB Inc. and others. All rights reserved. +# http://nexb.com and https://github.com/nexB/vulnerablecode/ +# The VulnerableCode software is licensed under the Apache License version 2.0. +# Data generated with VulnerableCode require an acknowledgment. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# When you publish or redistribute any data created with VulnerableCode or any VulnerableCode +# derivative work, you must accompany this data with the following acknowledgment: +# +# Generated with VulnerableCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# VulnerableCode should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# VulnerableCode is a free software from nexB Inc. and others. +# Visit https://github.com/nexB/vulnerablecode/ for support and download. + +import os +import json +from dateutil import parser as dateparser +from packageurl import PackageURL + +from unittest import TestCase + +from vulnerabilities.data_source import Reference +from vulnerabilities.data_source import Advisory +from vulnerabilities.data_source import VulnerabilitySeverity +from vulnerabilities.severity_systems import scoring_systems +from vulnerabilities.importers.apache_httpd import ApacheHTTPDDataSource +from vulnerabilities.importers.apache_httpd import fetch_links + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, "test_data/apache_httpd/CVE-1999-1199.json") + + +class TestApacheHTTPDDataSource(TestCase): + @classmethod + def setUpClass(cls): + data_source_cfg = {"etags": {}} + cls.data_src = ApacheHTTPDDataSource(1, config=data_source_cfg) + with open(TEST_DATA) as f: + cls.nvd_data = json.load(f) + + def test_to_advisory(self): + expected_advisories = [ + Advisory( + summary="A serious problem exists when a client sends a large number of " + "headers with the same header name. Apache uses up memory faster than the " + "amount of memory required to simply store the received data itself. That " + "is, memory use increases faster and faster as more headers are received, " + "rather than increasing at a constant rate. This makes a denial of service " + "attack based on this method more effective than methods which cause Apache" + " to use memory at a constant rate, since the attacker has to send less data.", + impacted_package_urls=[ + PackageURL( + type="apache", + namespace=None, + name="httpd", + version="1.3.1", + qualifiers={}, + subpath=None, + ), + PackageURL( + type="apache", + namespace=None, + name="httpd", + version="1.3.0", + qualifiers={}, + subpath=None, + ), + ], + resolved_package_urls=[], + references=[ + Reference( + url="https://httpd.apache.org/security/json/CVE-1999-1199.json", + severities=[ + VulnerabilitySeverity( + system=scoring_systems["generic_textual"], + value="important", + ), + ], + reference_id="CVE-1999-1199", + ), + ], + vulnerability_id="CVE-1999-1199", + ) + ] + found_advisories = [self.data_src.to_advisory(self.nvd_data)] + found_advisories = list(map(Advisory.normalized, found_advisories)) + expected_advisories = list(map(Advisory.normalized, expected_advisories)) + assert sorted(found_advisories) == sorted(expected_advisories) diff --git a/vulnerabilities/tests/test_data/apache_httpd/CVE-1999-1199.json b/vulnerabilities/tests/test_data/apache_httpd/CVE-1999-1199.json new file mode 100644 index 000000000..e1e52127f --- /dev/null +++ b/vulnerabilities/tests/test_data/apache_httpd/CVE-1999-1199.json @@ -0,0 +1,86 @@ +{ + "data_type": "CVE", + "data_format": "MITRE", + "data_version": "4.0", + "generator": { + "engine": "xmltojsonmjc 1.0" + }, + "references": {}, + "timeline": [ + { + "time": "1998-09-23", + "lang": "eng", + "value": "1.3.2 released" + } + ], + "CNA_private": { + "owner": "httpd" + }, + "CVE_data_meta": { + "ASSIGNER": "security@apache.org", + "AKA": "", + "STATE": "PUBLIC", + "ID": "CVE-1999-1199", + "TITLE": "Multiple header Denial of Service vulnerability" + }, + "source": { + "defect": [], + "advisory": "", + "discovery": "UNKNOWN" + }, + "problemtype": { + "problemtype_data": [ + { + "description": [ + { + "lang": "eng", + "value": "Multiple header Denial of Service vulnerability" + } + ] + } + ] + }, + "description": { + "description_data": [ + { + "lang": "eng", + "value": "A serious problem exists when a client sends a large number of headers with the same header name. Apache uses up memory faster than the amount of memory required to simply store the received data itself. That is, memory use increases faster and faster as more headers are received, rather than increasing at a constant rate. This makes a denial of service attack based on this method more effective than methods which cause Apache to use memory at a constant rate, since the attacker has to send less data." + } + ] + }, + "impact": [ + { + "other": "important" + } + ], + "affects": { + "vendor": { + "vendor_data": [ + { + "vendor_name": "Apache Software Foundation", + "product": { + "product_data": [ + { + "product_name": "Apache HTTP Server", + "version": { + "version_data": [ + { + "version_name": "1.3", + "version_affected": "=", + "version_value": "1.3.1" + }, + { + "version_name": "1.3", + "version_affected": "=", + "version_value": "1.3.0" + } + ] + } + } + ] + } + } + ] + } + } +} \ No newline at end of file From c25b48c56bb2c3d7bfe7c932200f41de13629c74 Mon Sep 17 00:00:00 2001 From: AmitGupta7580 Date: Tue, 6 Apr 2021 11:25:41 +0530 Subject: [PATCH 3/9] Add new Severity System for Apache_httpd Severity Signed-off-by: AmitGupta7580 --- vulnerabilities/importers/apache_httpd.py | 2 +- vulnerabilities/severity_systems.py | 5 +++++ vulnerabilities/tests/test_apache_httpd.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/importers/apache_httpd.py b/vulnerabilities/importers/apache_httpd.py index c5da0648f..54820e727 100644 --- a/vulnerabilities/importers/apache_httpd.py +++ b/vulnerabilities/importers/apache_httpd.py @@ -67,7 +67,7 @@ def to_advisory(self, data): description = data["description"]["description_data"] summary = next((item["value"] for item in description if item["lang"] == "eng"), "") severity = VulnerabilitySeverity( - system=scoring_systems["generic_textual"], + system=scoring_systems["apache_httpd"], value=data["impact"][0]["other"], ) reference = Reference(reference_id=cve, url=self.url + cve + ".json", severities=[severity]) diff --git a/vulnerabilities/severity_systems.py b/vulnerabilities/severity_systems.py index 33594268a..f8b7f9404 100644 --- a/vulnerabilities/severity_systems.py +++ b/vulnerabilities/severity_systems.py @@ -87,4 +87,9 @@ def as_score(self, value): url="", notes="Severity for unknown scoring systems. Contains generic textual values like High, Low etc", ), + "apache_httpd": ScoringSystem( + identifier="apache_httpd", + name="Apache Httpd Severity", + url="https://httpd.apache.org/security/impact_levels.html", + ), } diff --git a/vulnerabilities/tests/test_apache_httpd.py b/vulnerabilities/tests/test_apache_httpd.py index 3f28c59be..72b335937 100644 --- a/vulnerabilities/tests/test_apache_httpd.py +++ b/vulnerabilities/tests/test_apache_httpd.py @@ -80,7 +80,7 @@ def test_to_advisory(self): url="https://httpd.apache.org/security/json/CVE-1999-1199.json", severities=[ VulnerabilitySeverity( - system=scoring_systems["generic_textual"], + system=scoring_systems["apache_httpd"], value="important", ), ], From 2bc3acc7996063fc1a1d9342362d345fdd8e2556 Mon Sep 17 00:00:00 2001 From: AmitGupta7580 Date: Thu, 15 Apr 2021 14:52:13 +0530 Subject: [PATCH 4/9] Improve the format of apache_httpd importer Signed-off-by: AmitGupta7580 --- vulnerabilities/importers/apache_httpd.py | 52 ++++++++++++++++------ vulnerabilities/tests/test_apache_httpd.py | 16 ++----- 2 files changed, 42 insertions(+), 26 deletions(-) diff --git a/vulnerabilities/importers/apache_httpd.py b/vulnerabilities/importers/apache_httpd.py index 54820e727..17c5d44ca 100644 --- a/vulnerabilities/importers/apache_httpd.py +++ b/vulnerabilities/importers/apache_httpd.py @@ -22,9 +22,8 @@ import dataclasses -from bs4 import BeautifulSoup import requests -import re +from bs4 import BeautifulSoup from packageurl import PackageURL from vulnerabilities.data_source import Advisory @@ -44,7 +43,8 @@ class ApacheHTTPDDataSourceConfiguration(DataSourceConfiguration): class ApacheHTTPDDataSource(DataSource): CONFIG_CLASS = ApacheHTTPDDataSourceConfiguration - url = "https://httpd.apache.org/security/json/" + url = "https://httpd.apache.org/security/json/{}" + ref_url = "https://httpd.apache.org/security/json/{}.json" def updated_advisories(self): # Etags are like hashes of web responses. We maintain @@ -64,13 +64,37 @@ def updated_advisories(self): def to_advisory(self, data): cve = data["CVE_data_meta"]["ID"] - description = data["description"]["description_data"] - summary = next((item["value"] for item in description if item["lang"] == "eng"), "") - severity = VulnerabilitySeverity( - system=scoring_systems["apache_httpd"], - value=data["impact"][0]["other"], - ) - reference = Reference(reference_id=cve, url=self.url + cve + ".json", severities=[severity]) + descriptions = data.get("description", {}).get("description_data", []) + description = None + for desc in descriptions: + if desc.get("lang") == "eng": + description = desc.get("value") + break + + impacts = data.get("impact", []) + impact = None + for imp in impacts: + value = imp.get("other") + if value is not None: + impact = value + break + + if impact is not None: + severity = VulnerabilitySeverity( + system=scoring_systems["apache_httpd"], + value=impact, + ) + reference = Reference( + reference_id=cve, + url=self.ref_url.format(cve), + severities=[severity], + ) + else: + reference = Reference( + reference_id=cve, + url=self.ref_url.format(cve), + ) + resolved_packages = [] impacted_packages = [] @@ -91,7 +115,7 @@ def to_advisory(self, data): return Advisory( vulnerability_id=cve, - summary=summary, + summary=description, impacted_package_urls=impacted_packages, resolved_package_urls=resolved_packages, references=[reference], @@ -100,11 +124,11 @@ def to_advisory(self, data): def fetch_links(url): links = [] - data = requests.get(url).content + data = requests.get(url.format("")).content soup = BeautifulSoup(data, features="lxml") for tag in soup.find_all("a"): link = tag.get("href") - if not re.search("^CVE.*json$", link): + if not link.endswith("json"): continue - links.append(url + link) + links.append(url.format(link)) return links diff --git a/vulnerabilities/tests/test_apache_httpd.py b/vulnerabilities/tests/test_apache_httpd.py index 72b335937..a44577657 100644 --- a/vulnerabilities/tests/test_apache_httpd.py +++ b/vulnerabilities/tests/test_apache_httpd.py @@ -22,17 +22,15 @@ import os import json -from dateutil import parser as dateparser -from packageurl import PackageURL - from unittest import TestCase +from packageurl import PackageURL + from vulnerabilities.data_source import Reference from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import VulnerabilitySeverity from vulnerabilities.severity_systems import scoring_systems from vulnerabilities.importers.apache_httpd import ApacheHTTPDDataSource -from vulnerabilities.importers.apache_httpd import fetch_links BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data/apache_httpd/CVE-1999-1199.json") @@ -44,7 +42,7 @@ def setUpClass(cls): data_source_cfg = {"etags": {}} cls.data_src = ApacheHTTPDDataSource(1, config=data_source_cfg) with open(TEST_DATA) as f: - cls.nvd_data = json.load(f) + cls.data = json.load(f) def test_to_advisory(self): expected_advisories = [ @@ -59,19 +57,13 @@ def test_to_advisory(self): impacted_package_urls=[ PackageURL( type="apache", - namespace=None, name="httpd", version="1.3.1", - qualifiers={}, - subpath=None, ), PackageURL( type="apache", - namespace=None, name="httpd", version="1.3.0", - qualifiers={}, - subpath=None, ), ], resolved_package_urls=[], @@ -90,7 +82,7 @@ def test_to_advisory(self): vulnerability_id="CVE-1999-1199", ) ] - found_advisories = [self.data_src.to_advisory(self.nvd_data)] + found_advisories = [self.data_src.to_advisory(self.data)] found_advisories = list(map(Advisory.normalized, found_advisories)) expected_advisories = list(map(Advisory.normalized, expected_advisories)) assert sorted(found_advisories) == sorted(expected_advisories) From a4b4633723e33ba79bce540a92f80d4096442a2d Mon Sep 17 00:00:00 2001 From: AmitGupta7580 Date: Thu, 22 Apr 2021 20:13:54 +0530 Subject: [PATCH 5/9] rebased and refactored the apache_httpd importer according to the new model Signed-off-by: AmitGupta7580 --- vulnerabilities/importers/apache_httpd.py | 12 +++++----- vulnerabilities/tests/test_apache_httpd.py | 26 +++++++++++++--------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/vulnerabilities/importers/apache_httpd.py b/vulnerabilities/importers/apache_httpd.py index 17c5d44ca..44e590611 100644 --- a/vulnerabilities/importers/apache_httpd.py +++ b/vulnerabilities/importers/apache_httpd.py @@ -33,6 +33,7 @@ from vulnerabilities.data_source import VulnerabilitySeverity from vulnerabilities.severity_systems import scoring_systems from vulnerabilities.helpers import create_etag +from vulnerabilities.helpers import nearest_patched_package @dataclasses.dataclass @@ -95,8 +96,8 @@ def to_advisory(self, data): url=self.ref_url.format(cve), ) - resolved_packages = [] - impacted_packages = [] + fixed_packages = [] + affected_packages = [] for vendor in data["affects"]["vendor"]["vendor_data"]: for products in vendor["product"]["product_data"]: @@ -104,20 +105,19 @@ def to_advisory(self, data): version_value = version["version_value"] if version["version_affected"] == "<": - resolved_packages.append( + fixed_packages.append( PackageURL(type="apache", name="httpd", version=version_value) ) else: - impacted_packages.append( + affected_packages.append( PackageURL(type="apache", name="httpd", version=version_value) ) return Advisory( vulnerability_id=cve, summary=description, - impacted_package_urls=impacted_packages, - resolved_package_urls=resolved_packages, + affected_packages=nearest_patched_package(affected_packages, fixed_packages), references=[reference], ) diff --git a/vulnerabilities/tests/test_apache_httpd.py b/vulnerabilities/tests/test_apache_httpd.py index a44577657..12a774876 100644 --- a/vulnerabilities/tests/test_apache_httpd.py +++ b/vulnerabilities/tests/test_apache_httpd.py @@ -31,9 +31,10 @@ from vulnerabilities.data_source import VulnerabilitySeverity from vulnerabilities.severity_systems import scoring_systems from vulnerabilities.importers.apache_httpd import ApacheHTTPDDataSource +from vulnerabilities.helpers import AffectedPackage BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/apache_httpd/CVE-1999-1199.json") +TEST_DATA = os.path.join(BASE_DIR, "test_data", "apache_httpd", "CVE-1999-1199.json") class TestApacheHTTPDDataSource(TestCase): @@ -54,19 +55,22 @@ def test_to_advisory(self): "rather than increasing at a constant rate. This makes a denial of service " "attack based on this method more effective than methods which cause Apache" " to use memory at a constant rate, since the attacker has to send less data.", - impacted_package_urls=[ - PackageURL( - type="apache", - name="httpd", - version="1.3.1", + affected_packages=[ + AffectedPackage( + vulnerable_package=PackageURL( + type="apache", + name="httpd", + version="1.3.0", + ), ), - PackageURL( - type="apache", - name="httpd", - version="1.3.0", + AffectedPackage( + vulnerable_package=PackageURL( + type="apache", + name="httpd", + version="1.3.1", + ), ), ], - resolved_package_urls=[], references=[ Reference( url="https://httpd.apache.org/security/json/CVE-1999-1199.json", From 28f44ef9e052049089eda776a41c33dd20c2ad20 Mon Sep 17 00:00:00 2001 From: AmitGupta7580 Date: Sun, 25 Apr 2021 16:09:39 +0530 Subject: [PATCH 6/9] Resolve version ranges into discrete versions and fetch all released versions of Apache-httpd from Github API Signed-off-by: AmitGupta7580 --- vulnerabilities/importers/apache_httpd.py | 89 +++++++++++++++------- vulnerabilities/tests/test_apache_httpd.py | 32 ++++++++ 2 files changed, 95 insertions(+), 26 deletions(-) diff --git a/vulnerabilities/importers/apache_httpd.py b/vulnerabilities/importers/apache_httpd.py index 44e590611..25cfd5615 100644 --- a/vulnerabilities/importers/apache_httpd.py +++ b/vulnerabilities/importers/apache_httpd.py @@ -21,16 +21,20 @@ # Visit https://github.com/nexB/vulnerablecode/ for support and download. import dataclasses +import asyncio import requests from bs4 import BeautifulSoup from packageurl import PackageURL +from univers.versions import MavenVersion +from univers.version_specifier import VersionSpecifier from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource from vulnerabilities.data_source import DataSourceConfiguration from vulnerabilities.data_source import Reference from vulnerabilities.data_source import VulnerabilitySeverity +from vulnerabilities.package_managers import GitHubTagsAPI from vulnerabilities.severity_systems import scoring_systems from vulnerabilities.helpers import create_etag from vulnerabilities.helpers import nearest_patched_package @@ -47,6 +51,10 @@ class ApacheHTTPDDataSource(DataSource): url = "https://httpd.apache.org/security/json/{}" ref_url = "https://httpd.apache.org/security/json/{}.json" + def set_api(self): + self.version_api = GitHubTagsAPI() + asyncio.run(self.version_api.load_api(["apache/httpd"])) + def updated_advisories(self): # Etags are like hashes of web responses. We maintain # (url, etag) mappings in the DB. `create_etag` creates @@ -55,6 +63,7 @@ def updated_advisories(self): if create_etag(data_src=self, url=self.url, etag_key="ETag"): links = fetch_links(self.url) + self.set_api() advisories = [] for link in links: data = requests.get(link).json() @@ -72,6 +81,7 @@ def to_advisory(self, data): description = desc.get("value") break + severities = [] impacts = data.get("impact", []) impact = None for imp in impacts: @@ -79,40 +89,47 @@ def to_advisory(self, data): if value is not None: impact = value break - if impact is not None: - severity = VulnerabilitySeverity( - system=scoring_systems["apache_httpd"], - value=impact, - ) - reference = Reference( - reference_id=cve, - url=self.ref_url.format(cve), - severities=[severity], - ) - else: - reference = Reference( - reference_id=cve, - url=self.ref_url.format(cve), + severities.append( + VulnerabilitySeverity( + system=scoring_systems["apache_httpd"], + value=impact, + ) ) + reference = Reference( + reference_id=cve, + url=self.ref_url.format(cve), + severities=severities, + ) - fixed_packages = [] - affected_packages = [] - + versions = [] for vendor in data["affects"]["vendor"]["vendor_data"]: for products in vendor["product"]["product_data"]: for version in products["version"]["version_data"]: - version_value = version["version_value"] + versions.append(version) - if version["version_affected"] == "<": - fixed_packages.append( - PackageURL(type="apache", name="httpd", version=version_value) - ) + fixed_version_ranges, affected_version_ranges = self.to_version_ranges(versions) - else: - affected_packages.append( - PackageURL(type="apache", name="httpd", version=version_value) - ) + affected_packages = [] + fixed_packages = [] + + for version_range in fixed_version_ranges: + fixed_packages.extend( + [ + PackageURL(type="apache", name="httpd", version=version) + for version in self.version_api.get("apache/httpd") + if MavenVersion(version) in version_range + ] + ) + + for version_range in affected_version_ranges: + affected_packages.extend( + [ + PackageURL(type="apache", name="httpd", version=version) + for version in self.version_api.get("apache/httpd") + if MavenVersion(version) in version_range + ] + ) return Advisory( vulnerability_id=cve, @@ -121,6 +138,26 @@ def to_advisory(self, data): references=[reference], ) + def to_version_ranges(self, versions): + fixed_version_ranges = [] + affected_version_ranges = [] + for version in versions: + version_value = version["version_value"] + if version["version_affected"] == "<": + fixed_version_ranges.append( + VersionSpecifier.from_scheme_version_spec_string( + "maven", ">={}".format(version_value) + ) + ) + elif version["version_affected"] == "=" or version["version_affected"] == "?=": + affected_version_ranges.append( + VersionSpecifier.from_scheme_version_spec_string( + "maven", "{}".format(version_value) + ) + ) + + return (fixed_version_ranges, affected_version_ranges) + def fetch_links(url): links = [] diff --git a/vulnerabilities/tests/test_apache_httpd.py b/vulnerabilities/tests/test_apache_httpd.py index 12a774876..98e808634 100644 --- a/vulnerabilities/tests/test_apache_httpd.py +++ b/vulnerabilities/tests/test_apache_httpd.py @@ -25,10 +25,12 @@ from unittest import TestCase from packageurl import PackageURL +from univers.version_specifier import VersionSpecifier from vulnerabilities.data_source import Reference from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import VulnerabilitySeverity +from vulnerabilities.package_managers import GitHubTagsAPI from vulnerabilities.severity_systems import scoring_systems from vulnerabilities.importers.apache_httpd import ApacheHTTPDDataSource from vulnerabilities.helpers import AffectedPackage @@ -42,9 +44,39 @@ class TestApacheHTTPDDataSource(TestCase): def setUpClass(cls): data_source_cfg = {"etags": {}} cls.data_src = ApacheHTTPDDataSource(1, config=data_source_cfg) + known_versions = ["1.3.2", "1.3.1", "1.3.0"] + cls.data_src.version_api = GitHubTagsAPI(cache={"apache/httpd": known_versions}) with open(TEST_DATA) as f: cls.data = json.load(f) + def test_to_version_ranges(self): + versions = [ + { + "version_affected": "?=", + "version_value": "1.3.0", + }, + { + "version_affected": "=", + "version_value": "1.3.1", + }, + { + "version_affected": "<", + "version_value": "1.3.2", + }, + ] + fixed_version_ranges, affected_version_ranges = self.data_src.to_version_ranges(versions) + + # Check fixed packages + assert [ + VersionSpecifier.from_scheme_version_spec_string("maven", ">=1.3.2") + ] == fixed_version_ranges + + # Check vulnerable packages + assert [ + VersionSpecifier.from_scheme_version_spec_string("maven", "==1.3.0"), + VersionSpecifier.from_scheme_version_spec_string("maven", "==1.3.1"), + ] == affected_version_ranges + def test_to_advisory(self): expected_advisories = [ Advisory( From 6537967f88e46d50cb5ad40a5582c3252f33673b Mon Sep 17 00:00:00 2001 From: AmitGupta7580 Date: Wed, 28 Apr 2021 15:26:32 +0530 Subject: [PATCH 7/9] Resolve the problem of adding format string in etag and improve the code format Signed-off-by: AmitGupta7580 --- vulnerabilities/importers/apache_httpd.py | 54 +++++++++++----------- vulnerabilities/tests/test_apache_httpd.py | 4 +- 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/vulnerabilities/importers/apache_httpd.py b/vulnerabilities/importers/apache_httpd.py index 25cfd5615..aa4733c0e 100644 --- a/vulnerabilities/importers/apache_httpd.py +++ b/vulnerabilities/importers/apache_httpd.py @@ -20,8 +20,9 @@ # VulnerableCode is a free software tool from nexB Inc. and others. # Visit https://github.com/nexB/vulnerablecode/ for support and download. -import dataclasses import asyncio +import dataclasses +import urllib import requests from bs4 import BeautifulSoup @@ -48,8 +49,7 @@ class ApacheHTTPDDataSourceConfiguration(DataSourceConfiguration): class ApacheHTTPDDataSource(DataSource): CONFIG_CLASS = ApacheHTTPDDataSourceConfiguration - url = "https://httpd.apache.org/security/json/{}" - ref_url = "https://httpd.apache.org/security/json/{}.json" + base_url = "https://httpd.apache.org/security/json/" def set_api(self): self.version_api = GitHubTagsAPI() @@ -61,8 +61,8 @@ def updated_advisories(self): # (url, etag) pair. If a (url, etag) already exists then the code # skips processing the response further to avoid duplicate work - if create_etag(data_src=self, url=self.url, etag_key="ETag"): - links = fetch_links(self.url) + if create_etag(data_src=self, url=self.base_url, etag_key="ETag"): + links = fetch_links(self.base_url) self.set_api() advisories = [] for link in links: @@ -83,32 +83,29 @@ def to_advisory(self, data): severities = [] impacts = data.get("impact", []) - impact = None - for imp in impacts: - value = imp.get("other") + for impact in impacts: + value = impact.get("other") if value is not None: - impact = value - break - if impact is not None: - severities.append( - VulnerabilitySeverity( - system=scoring_systems["apache_httpd"], - value=impact, + severities.append( + VulnerabilitySeverity( + system=scoring_systems["apache_httpd"], + value=value, + ) ) - ) + break reference = Reference( reference_id=cve, - url=self.ref_url.format(cve), + url=urllib.parse.urljoin(self.base_url, f"{cve}.json"), severities=severities, ) - versions = [] + versions_data = [] for vendor in data["affects"]["vendor"]["vendor_data"]: for products in vendor["product"]["product_data"]: - for version in products["version"]["version_data"]: - versions.append(version) + for version_data in products["version"]["version_data"]: + versions_data.append(version_data) - fixed_version_ranges, affected_version_ranges = self.to_version_ranges(versions) + fixed_version_ranges, affected_version_ranges = self.to_version_ranges(versions_data) affected_packages = [] fixed_packages = [] @@ -138,18 +135,19 @@ def to_advisory(self, data): references=[reference], ) - def to_version_ranges(self, versions): + def to_version_ranges(self, versions_data): fixed_version_ranges = [] affected_version_ranges = [] - for version in versions: - version_value = version["version_value"] - if version["version_affected"] == "<": + for version_data in versions_data: + version_value = version_data["version_value"] + range_expression = version_data["version_affected"] + if range_expression == "<": fixed_version_ranges.append( VersionSpecifier.from_scheme_version_spec_string( "maven", ">={}".format(version_value) ) ) - elif version["version_affected"] == "=" or version["version_affected"] == "?=": + elif range_expression == "=" or range_expression == "?=": affected_version_ranges.append( VersionSpecifier.from_scheme_version_spec_string( "maven", "{}".format(version_value) @@ -161,11 +159,11 @@ def to_version_ranges(self, versions): def fetch_links(url): links = [] - data = requests.get(url.format("")).content + data = requests.get(url).content soup = BeautifulSoup(data, features="lxml") for tag in soup.find_all("a"): link = tag.get("href") if not link.endswith("json"): continue - links.append(url.format(link)) + links.append(urllib.parse.urljoin(url, link)) return links diff --git a/vulnerabilities/tests/test_apache_httpd.py b/vulnerabilities/tests/test_apache_httpd.py index 98e808634..384696646 100644 --- a/vulnerabilities/tests/test_apache_httpd.py +++ b/vulnerabilities/tests/test_apache_httpd.py @@ -50,7 +50,7 @@ def setUpClass(cls): cls.data = json.load(f) def test_to_version_ranges(self): - versions = [ + data = [ { "version_affected": "?=", "version_value": "1.3.0", @@ -64,7 +64,7 @@ def test_to_version_ranges(self): "version_value": "1.3.2", }, ] - fixed_version_ranges, affected_version_ranges = self.data_src.to_version_ranges(versions) + fixed_version_ranges, affected_version_ranges = self.data_src.to_version_ranges(data) # Check fixed packages assert [ From 4157037965cf88abd4ebda56dc26306eab8de242 Mon Sep 17 00:00:00 2001 From: AmitGupta7580 Date: Thu, 29 Apr 2021 18:52:30 +0530 Subject: [PATCH 8/9] Improve data extraction from JSON data Signed-off-by: AmitGupta7580 --- vulnerabilities/importers/apache_httpd.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vulnerabilities/importers/apache_httpd.py b/vulnerabilities/importers/apache_httpd.py index aa4733c0e..a5d720475 100644 --- a/vulnerabilities/importers/apache_httpd.py +++ b/vulnerabilities/importers/apache_httpd.py @@ -74,10 +74,10 @@ def updated_advisories(self): def to_advisory(self, data): cve = data["CVE_data_meta"]["ID"] - descriptions = data.get("description", {}).get("description_data", []) + descriptions = data["description"]["description_data"] description = None for desc in descriptions: - if desc.get("lang") == "eng": + if desc["lang"] == "eng": description = desc.get("value") break @@ -85,7 +85,7 @@ def to_advisory(self, data): impacts = data.get("impact", []) for impact in impacts: value = impact.get("other") - if value is not None: + if value: severities.append( VulnerabilitySeverity( system=scoring_systems["apache_httpd"], From 223cabedac3cf965a32236550bf2a0103f7fa679 Mon Sep 17 00:00:00 2001 From: AmitGupta7580 Date: Fri, 7 May 2021 00:00:42 +0530 Subject: [PATCH 9/9] Removed Etag check Signed-off-by: AmitGupta7580 --- vulnerabilities/importers/apache_httpd.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/vulnerabilities/importers/apache_httpd.py b/vulnerabilities/importers/apache_httpd.py index a5d720475..0b6e87bea 100644 --- a/vulnerabilities/importers/apache_httpd.py +++ b/vulnerabilities/importers/apache_httpd.py @@ -56,21 +56,13 @@ def set_api(self): asyncio.run(self.version_api.load_api(["apache/httpd"])) def updated_advisories(self): - # Etags are like hashes of web responses. We maintain - # (url, etag) mappings in the DB. `create_etag` creates - # (url, etag) pair. If a (url, etag) already exists then the code - # skips processing the response further to avoid duplicate work - - if create_etag(data_src=self, url=self.base_url, etag_key="ETag"): - links = fetch_links(self.base_url) - self.set_api() - advisories = [] - for link in links: - data = requests.get(link).json() - advisories.append(self.to_advisory(data)) - return self.batch_advisories(advisories) - - return [] + links = fetch_links(self.base_url) + self.set_api() + advisories = [] + for link in links: + data = requests.get(link).json() + advisories.append(self.to_advisory(data)) + return self.batch_advisories(advisories) def to_advisory(self, data): cve = data["CVE_data_meta"]["ID"]