From b2c73608a09ee068c191dad595c53113338345f7 Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Tue, 23 Mar 2021 02:19:43 +0530 Subject: [PATCH 1/5] [WIP] Collect Mattermost Signed-off-by: Hritik Vijay --- vulnerabilities/importers/mattermost.py | 48 +++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 vulnerabilities/importers/mattermost.py diff --git a/vulnerabilities/importers/mattermost.py b/vulnerabilities/importers/mattermost.py new file mode 100644 index 000000000..8695227a0 --- /dev/null +++ b/vulnerabilities/importers/mattermost.py @@ -0,0 +1,48 @@ +import dataclasses + +from bs4 import BeautifulSoup +from packageurl import PackageURL +import requests +from urllib.parse import urljoin + +from vulnerabilities.data_source import Advisory +from vulnerabilities.data_source import DataSource +from vulnerabilities.data_source import Reference + +SECURITY_UPDATES_URL = "https://mattermost.com/security-updates" + + +class MattermostDataSource(DataSource): + + def updated_advisories(self): + #TODO: Add etags + data = requests.get(SECURITY_UPDATES_URL).content + return self.batch_advisories(self.to_advisories(data)) + + def to_advisories(self, data): + advisories = [] + #FIXME: Change after this https://forum.mattermost.org/t/mattermost-website-returning-403-when-headers-contain-the-word-python/11412 + soup = BeautifulSoup(data, features="lxml", headers={'user-agent': 'aboutcode/vulnerablecode'}) + for row in soup.table.tbody.find_all('tr'): + ref_col, severity_score_col, affected_col,_, fixed_col, desc_col , name_col = row.select("td") + summary = desc_col.text + + affected_packages = [ + PackageURL( + type="mattermost", + name=name_col, + version=version.strip(), + qualifiers=pkg_qualifiers, + ) + for version in affected_col.text.split(",") #TODO: Not so easy + ] + + fixed_packages = [ + PackageURL( + type="generic", + name="postgresql", + version=version.strip(), + qualifiers=pkg_qualifiers, + ) + for version in fixed_col.text.split(",") + ] From 39c9a903f3ce8abc2821865288dd2e71fcc99980 Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Thu, 25 Mar 2021 06:29:58 +0530 Subject: [PATCH 2/5] Collect mattermost The code is not very elegant but so is the mattermost advisory page. They have been trying to make advisories more formatted and standard but at the time being, the current script should hold, hopefully in future too. Remaining tasks - [] Troubleshoot duplicate entries - As there is no vulnerability id on the page, everytime the importer is run it duplicates all the vulnerabilities. - [] Write tests Signed-off-by: Hritik Vijay --- SOURCES.rst | 2 + vulnerabilities/importer_yielder.py | 6 + vulnerabilities/importers/__init__.py | 1 + vulnerabilities/importers/mattermost.py | 190 +++++++++++++++++++++--- 4 files changed, 177 insertions(+), 22 deletions(-) diff --git a/SOURCES.rst b/SOURCES.rst index 281f77182..85de74563 100644 --- a/SOURCES.rst +++ b/SOURCES.rst @@ -47,3 +47,5 @@ +----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+ |suse_scores | https://ftp.suse.com/pub/projects/security/yaml/suse-cvss-scores.yaml |vulnerability severity scores by SUSE | +----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+ +|mattermost | https://mattermost.com/security-updates/ |mattermost server, desktop and mobile apps | ++----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+ diff --git a/vulnerabilities/importer_yielder.py b/vulnerabilities/importer_yielder.py index ebcd51079..9ad6c25c9 100644 --- a/vulnerabilities/importer_yielder.py +++ b/vulnerabilities/importer_yielder.py @@ -234,6 +234,12 @@ "data_source": "IstioDataSource", "data_source_cfg": {"repository_url": "https://github.com/istio/istio.io"}, }, + { + "name": "mattermost", + "license": "", + "data_source": "MattermostDataSource", + "data_source_cfg": {}, + }, ] diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index e7bbf70af..cb08e32b6 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -49,3 +49,4 @@ from vulnerabilities.importers.ubuntu_usn import UbuntuUSNDataSource from vulnerabilities.importers.apache_tomcat import ApacheTomcatDataSource from vulnerabilities.importers.istio import IstioDataSource +from vulnerabilities.importers.mattermost import MattermostDataSource diff --git a/vulnerabilities/importers/mattermost.py b/vulnerabilities/importers/mattermost.py index 8695227a0..d73a2a572 100644 --- a/vulnerabilities/importers/mattermost.py +++ b/vulnerabilities/importers/mattermost.py @@ -1,48 +1,194 @@ -import dataclasses +import re +from typing import List, Tuple +import asyncio from bs4 import BeautifulSoup +from dephell_specifier import RangeSpecifier from packageurl import PackageURL import requests -from urllib.parse import urljoin from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource from vulnerabilities.data_source import Reference +from vulnerabilities.data_source import VulnerabilitySeverity +from vulnerabilities.severity_systems import scoring_systems +from vulnerabilities.package_managers import GitHubTagsAPI SECURITY_UPDATES_URL = "https://mattermost.com/security-updates" +MM_REPO = { + "Mattermost Mobile Apps": "mattermost/mattermost-mobile", + "Mattermost Server": "mattermost/mattermost-server", + "Mattermost Desktop App": "mattermost/desktop", +} class MattermostDataSource(DataSource): - def updated_advisories(self): - #TODO: Add etags - data = requests.get(SECURITY_UPDATES_URL).content + # FIXME: Change after this https://forum.mattermost.org/t/mattermost-website-returning-403-when-headers-contain-the-word-python/11412 + self.set_api() + data = requests.get( + SECURITY_UPDATES_URL, headers={"user-agent": "aboutcode/vulnerablecode"} + ).content return self.batch_advisories(self.to_advisories(data)) + def set_api(self): + self.version_api = GitHubTagsAPI() + asyncio.run( + self.version_api.load_api( + [ + MM_REPO["Mattermost Mobile Apps"], + MM_REPO["Mattermost Server"], + MM_REPO["Mattermost Desktop App"], + ] + ) + ) + def to_advisories(self, data): advisories = [] - #FIXME: Change after this https://forum.mattermost.org/t/mattermost-website-returning-403-when-headers-contain-the-word-python/11412 - soup = BeautifulSoup(data, features="lxml", headers={'user-agent': 'aboutcode/vulnerablecode'}) - for row in soup.table.tbody.find_all('tr'): - ref_col, severity_score_col, affected_col,_, fixed_col, desc_col , name_col = row.select("td") - summary = desc_col.text + soup = BeautifulSoup(data, features="lxml") + for row in soup.table.tbody.find_all("tr"): + ( + ref_col, + severity_col, + affected_col, + _, + fixed_col, + desc_col, + name_col, + ) = row.select("td") - affected_packages = [ + name = name_col.text.strip() + if name not in MM_REPO: + continue + + fixed_versions = split_versions(fixed_col.text) + fixed_packages = [ PackageURL( type="mattermost", - name=name_col, - version=version.strip(), - qualifiers=pkg_qualifiers, + name=name, + version=version, ) - for version in affected_col.text.split(",") #TODO: Not so easy + for version in fixed_versions ] - fixed_packages = [ - PackageURL( - type="generic", - name="postgresql", - version=version.strip(), - qualifiers=pkg_qualifiers, + ( + affected_version_ranges, + excluded_version_ranges, + ) = to_affected_version_ranges(affected_col.text, fixed_col.text) + + affected_packages = [ + PackageURL(type="mattermost", name=name, version=version) + for version in self.version_api.get(MM_REPO[name]) + if + # The versions comparisions and advisories are not compatible with cloud-* versions + not version.startswith("cloud-") + and any((version in version_range for version_range in affected_version_ranges)) + and not any((version in version_range for version_range in excluded_version_ranges)) + ] + + """ + Severities are either "na" or cvssv3.1_qr + """ + references = [ + Reference( + reference_id=ref_col.text, + url=SECURITY_UPDATES_URL, + severities=[ + VulnerabilitySeverity( + system=scoring_systems["cvssv3.1_qr"], value=severity_col.text + ) + ] + if severity_col.text.lower() != "na" + else [], ) - for version in fixed_col.text.split(",") ] + + for cve_id in re.findall(r"cve-\d+-\d+", desc_col.text, re.IGNORECASE): + references.append( + Reference( + reference_id=cve_id, + url=f"https://cve.mitre.org/cgi-bin/cvename.cgi?name={cve_id}", + ) + ) + advisories.append( + Advisory( + vulnerability_id="", + summary=desc_col.text, + references=references, + impacted_package_urls=affected_packages, + resolved_package_urls=fixed_packages, + ) + ) + return advisories + + +def split_versions(versions: str) -> List[str]: + """ + The versions can take the form: + - v1.2,2.2 and 3.2 -> [1.2,2.2,3.2] + - v1 and v2 -> [1,2] + - v1, v2 -> [1,2] + - <10 -> [<10] + - na -> [] + - all -> all (see `affected_version_ranges`) + Returns list of versions without leading 'v' + """ + versions = versions.lower().strip().replace("and", ",") + if versions == "na": + return [] + if versions == "all": + return ["all"] + + versions = [ + # some versions are like v2.4, remove v + version.strip().replace("v", "") + for version in versions.split(",") + if version.strip() + ] + return versions + + +def to_affected_version_ranges( + affected_col: str, fixed_col: str +) -> Tuple[List[RangeSpecifier], List[RangeSpecifier]]: + """ + affected_col could be of type "v5.20.x to v5.26.x, excluding v5.25.5 and v5.26.2" + fixed_col is only relevent in case affected_col is "all" + "all" means all the versions before the only present fixed. If there are many fixed versions, it doesn't return anything. + Needs to be improved after https://github.com/nexB/vulnerablecode/issues/119 + According to https://forum.mattermost.org/t/all-affected-versions-in-the-mattermost-advisory/11423, + + Returns affected version included_ranges, excluded_ranges + """ + fixed_versions = split_versions(fixed_col) + affected_col = affected_col.replace(".x", ".*") # For 5.20.x + included, *excluded = affected_col.split("excluding") + range_expressions = split_versions(included) + + if len(range_expressions) == 1: + # special cases + if range_expressions[0] == "na": + return [], [] + + if range_expressions[0] == "all": + if len(fixed_versions) > 1: + # it gets very complicated. see link above + return [], [RangeSpecifier()] + return [RangeSpecifier(f"<{fixed_versions[0]}")], [] + + included_ranges = [] + for range_expression in range_expressions: + if "to" in range_expression: + # eg range_expression == "3.2.0 to 3.2.1" + lower_bound, upper_bound = range_expression.split("to") + lower_bound = f">={lower_bound}" + upper_bound = f"<={upper_bound}" + included_ranges.append(RangeSpecifier(f"{lower_bound},{upper_bound}")) + else: + included_ranges.append(RangeSpecifier(range_expression)) + + excluded_ranges = [] + if len(excluded): + excluded_ranges = [RangeSpecifier(v) for v in split_versions(excluded[0])] + + return included_ranges, excluded_ranges From 4df1bba796fab3f22cc38475247bf6a809d96590 Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Thu, 25 Mar 2021 07:14:39 +0530 Subject: [PATCH 3/5] Add last_run key Signed-off-by: Hritik Vijay --- vulnerabilities/importer_yielder.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vulnerabilities/importer_yielder.py b/vulnerabilities/importer_yielder.py index 9ad6c25c9..c053c470f 100644 --- a/vulnerabilities/importer_yielder.py +++ b/vulnerabilities/importer_yielder.py @@ -237,6 +237,7 @@ { "name": "mattermost", "license": "", + "last_run": None, "data_source": "MattermostDataSource", "data_source_cfg": {}, }, From 590bb9167ff79ad57ecb4592f5f2317541e5ebbc Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Tue, 30 Mar 2021 15:48:36 +0530 Subject: [PATCH 4/5] Sort imports and remove improper docstring Signed-off-by: Hritik Vijay --- vulnerabilities/importers/mattermost.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/vulnerabilities/importers/mattermost.py b/vulnerabilities/importers/mattermost.py index d73a2a572..290281ea2 100644 --- a/vulnerabilities/importers/mattermost.py +++ b/vulnerabilities/importers/mattermost.py @@ -1,18 +1,19 @@ -import re -from typing import List, Tuple import asyncio +import re +from typing import List +from typing import Tuple +import requests from bs4 import BeautifulSoup from dephell_specifier import RangeSpecifier from packageurl import PackageURL -import requests from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import DataSource from vulnerabilities.data_source import Reference from vulnerabilities.data_source import VulnerabilitySeverity -from vulnerabilities.severity_systems import scoring_systems from vulnerabilities.package_managers import GitHubTagsAPI +from vulnerabilities.severity_systems import scoring_systems SECURITY_UPDATES_URL = "https://mattermost.com/security-updates" MM_REPO = { @@ -86,9 +87,7 @@ def to_advisories(self, data): and not any((version in version_range for version_range in excluded_version_ranges)) ] - """ - Severities are either "na" or cvssv3.1_qr - """ + # Severities are either "na" or cvssv3.1_qr references = [ Reference( reference_id=ref_col.text, From 3f80316d207ad53e3e953f3ee7b5ceba1501f309 Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Tue, 8 Feb 2022 15:48:53 +0530 Subject: [PATCH 5/5] Ignore mattermost importer doctests Signed-off-by: Hritik Vijay --- pytest.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/pytest.ini b/pytest.ini index 7a196de40..289fe96a6 100644 --- a/pytest.ini +++ b/pytest.ini @@ -30,5 +30,6 @@ addopts = --ignore=vulnerabilities/importers/suse_backports.py --ignore=vulnerabilities/importers/suse_scores.py --ignore=vulnerabilities/importers/ubuntu_usn.py + --ignore=vulnerabilities/importers/mattermost.py --ignore=vulnerabilities/management/commands/create_cpe_to_purl_map.py --ignore=vulnerabilities/lib_oval.py