From e374aea1fe5c232579c217c9d910949b669370e8 Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Sat, 20 Mar 2021 05:33:55 +0530 Subject: [PATCH 01/16] [WIP] Collect Mozilla This is based on #78. Most of the work is done. Here's the checklist - [x] Parsing for mozilla data source - [x] Working mozilla importer - [ ] Migrate to GitDataSource - [ ] Solve TODOs - [ ] Write test cases Currently, it requires two extra dependencies: PyGithub and markdown I would eliminate the PyGithub dependency next but I think markdown has to stay. Final dependences would be comitted later. Signed-off-by: Hritik Vijay --- vulnerabilities/importer_yielder.py | 7 + vulnerabilities/importers/__init__.py | 1 + vulnerabilities/importers/mozilla.py | 180 ++++++++++++++++++++++++++ 3 files changed, 188 insertions(+) create mode 100644 vulnerabilities/importers/mozilla.py diff --git a/vulnerabilities/importer_yielder.py b/vulnerabilities/importer_yielder.py index ebcd51079..0e5c0fa9e 100644 --- a/vulnerabilities/importer_yielder.py +++ b/vulnerabilities/importer_yielder.py @@ -234,6 +234,13 @@ "data_source": "IstioDataSource", "data_source_cfg": {"repository_url": "https://github.com/istio/istio.io"}, }, + { + "name": "mozilla", + "license": "mpl-2.0", + "last_run": None, + "data_source": "MozillaDataSource", + "data_source_cfg": {}, + }, ] diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index e7bbf70af..3ab32983b 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -49,3 +49,4 @@ from vulnerabilities.importers.ubuntu_usn import UbuntuUSNDataSource from vulnerabilities.importers.apache_tomcat import ApacheTomcatDataSource from vulnerabilities.importers.istio import IstioDataSource +from vulnerabilities.importers.mozilla import MozillaDataSource diff --git a/vulnerabilities/importers/mozilla.py b/vulnerabilities/importers/mozilla.py new file mode 100644 index 000000000..5b1de2e90 --- /dev/null +++ b/vulnerabilities/importers/mozilla.py @@ -0,0 +1,180 @@ +from typing import Set, List + +import re +from bs4 import BeautifulSoup +from packageurl import PackageURL +import requests + +from github import Github +import yaml +from markdown import markdown + +from vulnerabilities.data_source import Advisory +from vulnerabilities.data_source import DataSource +from vulnerabilities.data_source import Reference +from vulnerabilities.data_source import VulnerabilitySeverity +from vulnerabilities.severity_systems import scoring_systems +from vulnerabilities.helpers import is_cve + + +REPOSITORY = "mozilla/foundation-security-advisories" +MFSA_FILENAME_RE = re.compile(r"mfsa(\d{4}-\d{2,3})\.(md|yml)$") + + +class MozillaDataSource(DataSource): + def updated_advisories(self) -> Set[Advisory]: + advisories = [] + advisory_links = self.fetch_advisory_links() + for link in advisory_links: + advisories.extend(self.to_advisories(link)) + return self.batch_advisories(advisories) + + def fetch_advisory_links(self): + links = [] + # TODO: Migrate to GitDataSource + g = Github("ffa10510de8dfa1bad60cd3963c45d2db2035287") + repo = g.get_repo(REPOSITORY) + years = repo.get_contents("announce") + for year in years: + if year.type != "dir": + continue + advisories = repo.get_contents(year.path) + links.extend([advisory.download_url for advisory in advisories]) + return links + + def to_advisories(self, link: str) -> Set[Advisory]: + advisories = [] + + if link.endswith(".md"): + advisories.extend(self.parse_md(link)) + elif link.endswith(".yml"): + advisories.extend(self.parse_yml(link)) + + return advisories + + def parse_yml(self, link) -> List[Advisory]: + advisories = [] + advisory_page = requests.get(link).text + data = yaml.safe_load(advisory_page) + + mfsa_id = self.mfsa_id_from_filename(link) + if mfsa_id: + data["mfsa_id"] = mfsa_id + else: + ValueError("mfsa_id not present") + # FIXME: Handle else case too? mfsa_id must be there anyway + + fixed_package_urls = self.get_package_urls(data["fixed_in"]) + references = self.get_references(data) + + for cve, advisory in data["advisories"].items(): + if not is_cve(cve): + continue + + advisories.append( + Advisory( + summary=advisory["description"], + vulnerability_id=cve, + impacted_package_urls=[], + resolved_package_urls=fixed_package_urls, + references=references, + ) + ) + + return advisories + + def parse_md(self, link) -> List[Advisory]: + advisory_page = requests.get(link).text + yamltext, mdtext = self.parse_md_front_matter(advisory_page) + + data = yaml.safe_load(yamltext) + mfsa_id = self.mfsa_id_from_filename(link) + if mfsa_id: + data["mfsa_id"] = mfsa_id + else: + ValueError("mfsa_id not present") + # FIXME: Same as parse_yml + + fixed_package_urls = self.get_package_urls(data["fixed_in"]) + references = self.get_references(data) + + soup = BeautifulSoup(markdown(mdtext), features="lxml") + + description = "" + descTag = soup.find("h3", text=lambda txt: txt.lower() == "description") + if descTag: + for tag in descTag.next_siblings: + if tag.name: + if tag.name != "p": + break + description += tag.get_text() + + # FIXME: add references from md ? They lack a proper reference id and are mostly bug reports + + return [ + Advisory( + summary=description, + vulnerability_id="", # FIXME: Scrape the entire page for CVE regex ? + impacted_package_urls=[], + resolved_package_urls=fixed_package_urls, + references=references, + ) + ] + + def parse_md_front_matter(self, lines): + """Return the YAML and MD sections. + :param: lines iterator + :return: str YAML, str Markdown + """ + # fm_count: 0: init, 1: in YAML, 2: in Markdown + lines = lines.split("\n") + fm_count = 0 + yaml_lines = [] + md_lines = [] + for line in lines: + # first line we care about is FM start + if fm_count < 2 and line.strip() == "---": + fm_count += 1 + continue + + if fm_count == 1: + yaml_lines.append(line) + + if fm_count == 2: + md_lines.append(line) + + return "\n".join(yaml_lines), "\n".join(md_lines) + + def mfsa_id_from_filename(self, filename): + match = MFSA_FILENAME_RE.search(filename) + if match: + return "mfsa" + match.group(1) + + return None + + def get_package_urls(self, pkgs: List[str]) -> List[PackageURL]: + package_urls = [ + PackageURL( + type="mozilla", + # TODO: Improve after https://github.com/mozilla/foundation-security-advisories/issues/76#issuecomment-803082182 + name=" ".join(pkg.split(" ")[0:-1]), + version=pkg.split(" ")[-1], + ) + for pkg in pkgs + ] + return package_urls + + def get_references(self, data: any) -> List[Reference]: + # FIXME: Should we add 'bugs' section in references too? + # Should we add 'impact'/severity of CVE in references too? + # If yes, then fix alpine_linux importer as well + # Otherwise, do we need severity fieild for adversary as well? + return [ + Reference( + reference_id=data["mfsa_id"], + url="https://www.mozilla.org/en-US/security/advisories/{}".format(data["mfsa_id"]), + severities=[ + VulnerabilitySeverity(scoring_systems["cvssv3.1_qr"], data.get("impact")) + ], + ) + ] From cdbb9898a4a771734d9f8ee9c63ed7bf098c860e Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Sat, 20 Mar 2021 23:51:27 +0530 Subject: [PATCH 02/16] Migrate to GitDataSource from PyGithub [x] Migrate to GitDataSource [x] Update dependencies [x] More verbose comments Signed-off-by: Hritik Vijay --- requirements.txt | 3 +- vulnerabilities/importer_yielder.py | 5 +- vulnerabilities/importers/mozilla.py | 178 ++++++++++++++++----------- 3 files changed, 109 insertions(+), 77 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6d45f57bb..af2048e25 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,6 +20,7 @@ ipython==7.13.0 ipython-genutils==0.2.0 jedi==0.17.0 lxml==4.6.3 +Markdown==3.3.4 more-itertools==8.0.2 packageurl-python==0.9.3 packaging==19.2 @@ -55,4 +56,4 @@ zipp==0.6.0 requests==2.23.0 toml==0.10.2 PyYAML==5.4 -freezegun==1.1.0 \ No newline at end of file +freezegun==1.1.0 diff --git a/vulnerabilities/importer_yielder.py b/vulnerabilities/importer_yielder.py index 0e5c0fa9e..dfe834224 100644 --- a/vulnerabilities/importer_yielder.py +++ b/vulnerabilities/importer_yielder.py @@ -239,7 +239,10 @@ "license": "mpl-2.0", "last_run": None, "data_source": "MozillaDataSource", - "data_source_cfg": {}, + "data_source_cfg": { + "branch": None, + "repository_url": "https://github.com/mozilla/foundation-security-advisories", + }, }, ] diff --git a/vulnerabilities/importers/mozilla.py b/vulnerabilities/importers/mozilla.py index 5b1de2e90..4149e92cb 100644 --- a/vulnerabilities/importers/mozilla.py +++ b/vulnerabilities/importers/mozilla.py @@ -1,19 +1,20 @@ -from typing import Set, List +from typing import Set, List, Generator import re +import asyncio from bs4 import BeautifulSoup from packageurl import PackageURL import requests -from github import Github import yaml from markdown import markdown from vulnerabilities.data_source import Advisory -from vulnerabilities.data_source import DataSource +from vulnerabilities.data_source import GitDataSource from vulnerabilities.data_source import Reference from vulnerabilities.data_source import VulnerabilitySeverity from vulnerabilities.severity_systems import scoring_systems +from vulnerabilities.package_managers import GitHubTagsAPI from vulnerabilities.helpers import is_cve @@ -21,51 +22,70 @@ MFSA_FILENAME_RE = re.compile(r"mfsa(\d{4}-\d{2,3})\.(md|yml)$") -class MozillaDataSource(DataSource): +class MozillaDataSource(GitDataSource): + def __enter__(self): + super(MozillaDataSource, self).__enter__() + + if not getattr(self, "_added_files", None): + self._added_files, self._updated_files = self.file_changes( + recursive=True, subdir="announce" + ) + + # Do we need this ? + # self.version_api = GitHubTagsAPI() + # self.set_api() + + def set_api(self): + repository = "/".join(self.config.repository_url.split("/")[-2:]) + asyncio.run(self.version_api.load_api([repository])) + + def added_advisories(self) -> Set[Advisory]: + return self._load_advisories(self._added_files) + def updated_advisories(self) -> Set[Advisory]: - advisories = [] - advisory_links = self.fetch_advisory_links() - for link in advisory_links: - advisories.extend(self.to_advisories(link)) - return self.batch_advisories(advisories) - - def fetch_advisory_links(self): - links = [] - # TODO: Migrate to GitDataSource - g = Github("ffa10510de8dfa1bad60cd3963c45d2db2035287") - repo = g.get_repo(REPOSITORY) - years = repo.get_contents("announce") - for year in years: - if year.type != "dir": - continue - advisories = repo.get_contents(year.path) - links.extend([advisory.download_url for advisory in advisories]) - return links + return self._load_advisories(self._updated_files) + + def _load_advisories(self, files) -> Set[Advisory]: + """ + Yields list of advisories of batch size + """ + files = [ + f for f in files if f.endswith(".md") or f.endswith(".yml") + ] # skip irrelevant files - def to_advisories(self, link: str) -> Set[Advisory]: advisories = [] + for path in files: + for advisory in self._to_advisories(path): + advisories.append(advisory) + if len(advisories) >= self.batch_size: + yield advisories + advisories = [] + + def _to_advisories(self, path: str) -> List[Advisory]: + """ + Convert a file to corresponding advisories. + This calls proper method to handle yml/md files. + """ + mfsa_id = self._mfsa_id_from_filename(path) - if link.endswith(".md"): - advisories.extend(self.parse_md(link)) - elif link.endswith(".yml"): - advisories.extend(self.parse_yml(link)) + with open(path) as lines: + if path.endswith(".md"): + return self._parse_md(mfsa_id, lines) + elif path.endswith(".yml"): + return self._parse_yml(mfsa_id, lines) - return advisories + return [] - def parse_yml(self, link) -> List[Advisory]: + def _parse_yml(self, mfsa_id, lines) -> List[Advisory]: advisories = [] - advisory_page = requests.get(link).text - data = yaml.safe_load(advisory_page) + data = yaml.safe_load(lines) + data["mfsa_id"] = mfsa_id - mfsa_id = self.mfsa_id_from_filename(link) - if mfsa_id: - data["mfsa_id"] = mfsa_id - else: - ValueError("mfsa_id not present") - # FIXME: Handle else case too? mfsa_id must be there anyway + fixed_package_urls = self._get_package_urls(data.get("fixed_in")) + references = self._get_references(data) - fixed_package_urls = self.get_package_urls(data["fixed_in"]) - references = self.get_references(data) + if not data.get("advisories"): + return [] for cve, advisory in data["advisories"].items(): if not is_cve(cve): @@ -73,7 +93,7 @@ def parse_yml(self, link) -> List[Advisory]: advisories.append( Advisory( - summary=advisory["description"], + summary=advisory.get("description"), vulnerability_id=cve, impacted_package_urls=[], resolved_package_urls=fixed_package_urls, @@ -83,31 +103,16 @@ def parse_yml(self, link) -> List[Advisory]: return advisories - def parse_md(self, link) -> List[Advisory]: - advisory_page = requests.get(link).text - yamltext, mdtext = self.parse_md_front_matter(advisory_page) + def _parse_md(self, mfsa_id, lines) -> List[Advisory]: + yamltext, mdtext = self._parse_md_front_matter(lines) data = yaml.safe_load(yamltext) - mfsa_id = self.mfsa_id_from_filename(link) - if mfsa_id: - data["mfsa_id"] = mfsa_id - else: - ValueError("mfsa_id not present") - # FIXME: Same as parse_yml - - fixed_package_urls = self.get_package_urls(data["fixed_in"]) - references = self.get_references(data) - - soup = BeautifulSoup(markdown(mdtext), features="lxml") - - description = "" - descTag = soup.find("h3", text=lambda txt: txt.lower() == "description") - if descTag: - for tag in descTag.next_siblings: - if tag.name: - if tag.name != "p": - break - description += tag.get_text() + data["mfsa_id"] = mfsa_id + + fixed_package_urls = self._get_package_urls(data.get("fixed_in")) + references = self._get_references(data) + + description = self._html_get_p_under_h3(markdown(mdtext), "description") # FIXME: add references from md ? They lack a proper reference id and are mostly bug reports @@ -121,13 +126,25 @@ def parse_md(self, link) -> List[Advisory]: ) ] - def parse_md_front_matter(self, lines): - """Return the YAML and MD sections. + def _html_get_p_under_h3(self, html, h3: str): + soup = BeautifulSoup(html, features="lxml") + h3tag = soup.find("h3", text=lambda txt: txt.lower() == h3) + p = "" + if h3tag: + for tag in h3tag.next_siblings: + if tag.name: + if tag.name != "p": + break + p += tag.get_text() + return p + + def _parse_md_front_matter(self, lines): + """ + Return the YAML and MD sections. :param: lines iterator :return: str YAML, str Markdown """ # fm_count: 0: init, 1: in YAML, 2: in Markdown - lines = lines.split("\n") fm_count = 0 yaml_lines = [] md_lines = [] @@ -143,20 +160,21 @@ def parse_md_front_matter(self, lines): if fm_count == 2: md_lines.append(line) - return "\n".join(yaml_lines), "\n".join(md_lines) + return "".join(yaml_lines), "".join(md_lines) - def mfsa_id_from_filename(self, filename): + def _mfsa_id_from_filename(self, filename): match = MFSA_FILENAME_RE.search(filename) if match: return "mfsa" + match.group(1) return None - def get_package_urls(self, pkgs: List[str]) -> List[PackageURL]: + def _get_package_urls(self, pkgs: List[str]) -> List[PackageURL]: package_urls = [ PackageURL( type="mozilla", # TODO: Improve after https://github.com/mozilla/foundation-security-advisories/issues/76#issuecomment-803082182 + # pkg is of the form "Firefox ESR 1.21" or "Thunderbird 2.21" name=" ".join(pkg.split(" ")[0:-1]), version=pkg.split(" ")[-1], ) @@ -164,17 +182,27 @@ def get_package_urls(self, pkgs: List[str]) -> List[PackageURL]: ] return package_urls - def get_references(self, data: any) -> List[Reference]: - # FIXME: Should we add 'bugs' section in references too? + def _get_references(self, data: any) -> List[Reference]: + """ + Returns a list of references + Currently only considers the given mfsa as a reference + """ + # FIXME: Needs improvement + # Should we add 'bugs' section in references too? # Should we add 'impact'/severity of CVE in references too? # If yes, then fix alpine_linux importer as well - # Otherwise, do we need severity fieild for adversary as well? + # Otherwise, do we need severity field for adversary as well? + + # FIXME: Write a helper for cvssv3.1_qr severity detection ? + severities = ["critical", "low", "high", "medium", "none"] + severity = [{severity in data.get("impact"): severity} for severity in severities][0].get( + True + ) + return [ Reference( reference_id=data["mfsa_id"], url="https://www.mozilla.org/en-US/security/advisories/{}".format(data["mfsa_id"]), - severities=[ - VulnerabilitySeverity(scoring_systems["cvssv3.1_qr"], data.get("impact")) - ], + severities=[VulnerabilitySeverity(scoring_systems["cvssv3.1_qr"], severity)], ) ] From 27423e7284cfd8ff4f3ee91c9532dfdf632e805f Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Sun, 21 Mar 2021 05:24:53 +0530 Subject: [PATCH 03/16] Edge case: handle large batch and vul w/o cve Signed-off-by: Hritik Vijay --- vulnerabilities/importers/mozilla.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vulnerabilities/importers/mozilla.py b/vulnerabilities/importers/mozilla.py index 4149e92cb..2f4662283 100644 --- a/vulnerabilities/importers/mozilla.py +++ b/vulnerabilities/importers/mozilla.py @@ -61,6 +61,9 @@ def _load_advisories(self, files) -> Set[Advisory]: yield advisories advisories = [] + # In case batch size is too high + yield advisories + def _to_advisories(self, path: str) -> List[Advisory]: """ Convert a file to corresponding advisories. @@ -88,13 +91,10 @@ def _parse_yml(self, mfsa_id, lines) -> List[Advisory]: return [] for cve, advisory in data["advisories"].items(): - if not is_cve(cve): - continue - advisories.append( Advisory( summary=advisory.get("description"), - vulnerability_id=cve, + vulnerability_id=cve if is_cve(cve) else "", impacted_package_urls=[], resolved_package_urls=fixed_package_urls, references=references, From c0faf6710666d69f758e0ae1871a8422b15cf20a Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Sun, 21 Mar 2021 05:34:54 +0530 Subject: [PATCH 04/16] Test cases for mozilla importer Signed-off-by: Hritik Vijay --- vulnerabilities/tests/test_data/mozilla.zip | Bin 0 -> 36012 bytes vulnerabilities/tests/test_mozilla.py | 98 ++++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 vulnerabilities/tests/test_data/mozilla.zip create mode 100644 vulnerabilities/tests/test_mozilla.py diff --git a/vulnerabilities/tests/test_data/mozilla.zip b/vulnerabilities/tests/test_data/mozilla.zip new file mode 100644 index 0000000000000000000000000000000000000000..a572c4e9aa1e884eb4b4acdf5ff836d05d29b925 GIT binary patch literal 36012 zcmbSzbzIcj);-5t_6%naSFgmjl6h;%E`4N7+-C0&AaNs82OIOo3a z`JMZo>pl0*=du(q=U zo4LVU-MDpi9v~n)zSMhmZ|+`L2#6?K$Os6(eQEqJjQ&5t5dJd^m!@gOdLxsFkCm?Y~DPhI561aE}3df_aFbQUK&#HXtpC~w|X zB0i&z2z>U22pql_x?gmI>8a))r!Wx@k>T1Fj* zKx%2N&cK;HN-%7=t39l-Bqd?7PkZLMux4gPu=u8|T={{}E zP3*VLI|rsLftSE;Jt>|-A=qpNp_T8sz~>;n=&onD*`KhaI$hY}I&WKNd?h4y4pi^n zlbvDRJKP`aZ~qv;zkoCE5BEp5U$75xw70i*`JIb$S-ReCx#>;3h)*6-U^MZQL=hZT~?jZb}! ze5f*2pTEGs^V;gcN9?--yKuxZQry~#P0B~&6{YG_Qy(2MmlEQTjjfYzALdbjGn|@3 zSZOE}Y`u#fW1xk7Xuhkwj`UYD( znZBD-kj;4o-($+H=pJ-&dtO< zZmRN1BbW&QV%*LC(6=J|f=I46{82G^FC%NY(1)Bl)3B;hn_aO?Q(G?TR*k6KL&~>T zQGS`p-@<7O!azN!kU~k`*}H0uV|fr|yUp5+eEB-^`1N{tJoIXm-1lwywIF+f!OV#z z5VN?5Icbc@{L+J6z&E@lzuf+6C20b5)CF87P~j12-uz%Y&zsMooUaaK5zz zzR*>?fi5^m1ZM&JxXlfc2uXX40|!e#h@|m6BvHX;Rn}c1 zk+zL5WyMsJ`t@FM?sj70A+qK7>S~c{Jk?O(c8yceh)ZV2|L9-9HmO4ITd1@>elDHg zY$C%kAjzCA{lTBzIs-uY;<1@ho~;pZ1s%f*cE=!D@Pe3#M&usxz;`+n-+^Tx`)GYeiWrUgQS# zX(Bg0iGqpJ_qk~vHzs?C+2tn4d_yTx&gqZIEE2l4gbBMU=Oqb}vSw1SxzFbKkU5%_ zve>GK0y@4EO)W^aylG>dlyOm=|0QVS6$axJB}U5{c7!UWH`DP0(c5Wjvc zUrH^%`J$YpV7E8DeLcb*bL+Fm8qtM;51(zUOKB?>SPvWt*t@8rn+WDOsezK;@w{23 z5_%L;)qD#>7dy+>Tqjcw~8+|1mvo{#faC@Z7*?Z583|*~Zl1*p`H2&;FNWEY!pe z>O+T|WSsb*;mY;P%l(AS&0xxMZrRe4?)(P;X}2?Chx9}XDH{!0R{^kkf=rZg`NGKg z!$(o3y{AH$r5YYY>?S5JL}@ckHVgW>1(APzPnBpK7ogIrrw=2HL@o-?nFdShW-|>s zjjQ5&?iRDvcT@8d7PpgPhyiTs$e*|*uh6O8hI*2FX*)6@LAD23c|Y9jg#|yL z&-zZDD3tr@Ga}6|np8>^5^oLvbViNEB*FLdN_K z3w1I0#n$FwrX|rHuT!#G?Mj2&5~*1n6Jj9S*9;o&l8SM&)Ne`ZG(RLEzq0SHY0>2J zpB=w^Bnk>4Iz}axwm&Pw2(_Gv4YOR2Up1h2J=J!MYJO?QXbFZuLO!br)u_K9i_5$@ zzdCs!0=o74?t-vb`mIx)huVg(1V7Ns^J<^;8#Vml#5(NQqevcbUvuYf6JvZdG(fGxh1}e zj6XS~2IGvLtsPA7!V&R7$Dw@?0lceTr}smqF^2mG%>c?FzsfVKYcC z=Q>%PK{BWpk&~NPT`Y#zMbwm#ItsQ*XwE*BKyk0vFx{K&;U^6n z&kYGZ9kPjnDtBUqebi{5%8^|8*_MGZfo$Cjd4sb8MTon4C?TEjD2I3PtN!D%^<>G{ zObcnCF)1hY)$NJ=vy;J18Ean*qBoTNF*cM(x}{s&W1M5uJYp{#QRoPdHI8|y4=)21 zd-8hl9>fxFM8>dR{vc=+{DF?RIE(Xq-|B-u?{Yakz*Z@QlIm;OA!-=4=f^gJD92tB zudm;Kyq6ebjcVqi>YONy9To8Oyeu{Oyfrs_rkh9krF>>8K2CK_ZLFk+@u7kaQcf0l z)rP2AyxHtzrKQ0g>6;bc_MB5`Z6G_f*`sE{X;j%ZMzwd}`;NQswxMC zW}WjB^1d&0HRhums%FrB#IDBg{D=ipRPEM*pSv1zb$>H`h1yuIHssCm7G0a@dr#<# z7k4Y!pJ`%S-dY7*e6kQ$)|m4dWjJ59GZ$nhXkB&`D?OTG8A#Iqq|fAi2qNm~Rx$7a zgpwX$56bHmj(Y=M{lM2>Ym*sD;YpG5%V;Yhb~O!gs?fY8VHv9TMUdQon^`Bd8qE7&P(Jq$roPJsG&M-7YmU zZ*XXob1TvNGj_P;4OCI0tDI-6F#}EV9O&-Gg08`Cyww)I>-6x z7)zsSqXQcLnC-ZylinYd>pe_*VO_#_8?^1-rri#uCdptVtNigN^d}F4H{3EP&4oog4pQ67INL!ph2i>DK1K zNRvHxXUg$2Wh?0@BQ(02q(=|}18`#DS0_nLu6%5%9P4is{87wgzKoIPf}u_N?1000 zjHd&7GK)X8Td+VA`e3t!jGwiXZ|hzXO?VOklg=Et`Gf?0F?^1Km9MIRk8m$irIkEW7CiREgLB1DsOzKN$8etVz>heS4APHE=&);2Kmq)zH=3CJ zbm;bxPRuy}_L}ivMM1*Vr>?SS129>)vy%#dBoBw|ILiK@HaDf2y#>(T*r<)(Ioq~N z;bHApAs3E%fH8l~z>%!%i{O6Oxv_gWCQyDpZ#RVZXTA@s0oc*->cAGB%l)cCxaa%# zD&cQ%ZLb4%oEN~qvAc;ZCSt;QCafr7;f~NbSb-pkUV?a@!--}nYLL#X#mpwc6?t>s zoCozp{HaPD=Xu!d?P=*)HvPq<_algX#A-X-%x#`urS`%o5>{c7yKddvu7^H1E&M7b zf4XJ#Zd#?l(I|~DO$bopHc^pt7Enl)*7J#N-~L1Y^KtM*{hn4jk=S(a2px*1i zN09g0%OI~v#dox4AG#1jAdZWnHH?ey?VjyBU6U@+FFK*@>+#r>s3^uNEqOY0^do$| zmg;+_l*HONEP>DD5L@TItHZvHOw?eN$RVBNR**c%YZIB;#((>oQugu&e969OMzv(} zsl|G0@uu58psFloE2fUCFeI3GL57RP@^s6t>LZ2b;;TwsSxCOqTAebwQo!5mpLARk zaiW`!sM4j7(ol9S>u}Me*2j;H{5K)8=*yP8!8kv^UKmV&>z!JISg@rt%BmrM&j`)j z4e-Gf8;!cgaCGA+rHOj8r0aovzWw>T`Mz%=JYqd+UlX6vKjXQSngzZ?t!xU)ibn~o*FjzGNStNraq{Ct z!&tbqO2i8jy6NUIoc2mP)-6!J{CpWK%9EsQ5SmS6jEau-jN*XJ<704^4b2gD^5heY z*UlC8RWSi}6$6h_Gg}FvN{*yJ5`%ZAAzWNt7<4{gwXJtn+04c$-_~=G1RcII$s=PV zKE+tma$TWluH6PG5w7C-nRzSNTcDs8ogiY{LAvEK#ebq@#JG98Qfz;WqmT2!e99O3 zC22PC?fkSUx;RoB_M?Wpp8Onyah(!iS`x%+%jYpoFfEIF(DfLX3|GFKQo%&(q+YJF;G z&vynlW`AZ->oitWDJ=169@C(a(`Ra45B+!x%Qfc|wfPV}%qEI3*Ik~(=kEp-y97;} z*dGO0#tH$JQU^JW1N}ecKK!}Na|=Z{K_O!pW-6bWr?ZtLcPgtZ=bTIC9AY+{4Jg@`vn;Ov%g_FKx=Sx?lcT?lKZ5rg!j#BpV)Hxqr2o zz~ycabFusnMDH7?t{%*T-+g#Ptj^<2*x=2Vn$k>pP}uf_D#u#TnBVEis3Xt*9K3^jtvqa4Ve`;7dpvj3F{3>hperk1U zgb!z{jFp93pH*FF&u?LG3vtyA0HU>@ zynDw#xS%OaVe zP>)2>P4fVjirSzV@LfP9D*bDt^z7sDFFUNwKB>H)m1iOcCEeyL#YM!|Gn5HB7C-kj zU7zAhuiso!eb##Q65(LRUf^YJyNMF%ZN19xlN)?`e>)^7FRM+G4o% z)KB&}GyI>LN5Omgn#<@22nZ+7q*DzOgY2fQ*wb{E{a9i6B}(M$am;_Q_*h%U%S!s3i%9-pz3}nqR4TKMRFsrq z(J)^xKPb1qYiu&9(lD}Pk-Uh{+KLm~>#X#YMfELsiHDcEik!}m8+nZ2D!IFkhB(ji=Qwg)nhpI9Em}DGr1~uRT4+{jA3*L^To3Y z-4}c@xkr1!wQ?Lw?Bd&au|GAF6O!6wULNJEKLW<#2Lg74Jk@?07FyH=LiwnMu3inr z*exycycjCfe=iypQ2W}?@41Jww5O+LT)>ksH|!whhp0>ywF>Xb*ykxT6xhw{l7#Yr zQMu$_+KJ!g;3dC@v~6$iZL{vs(8*sP?mi}N`uv1iTbc1IMx}R5a6ocmg2F4oC(Pd; zdg28j3FsbsYq61ZJh^?nl_V*!HO!r^tR8-){lz)Q_yC3&tp; zX_lwwDFtmeyXJ+6Qnu6B%7HZr9H3P5%zGpai<$fQwTfq z+;J!_8h3R%LymrYuia_q?^QNsFpb~7Jhip zuk+j@toq`c9rcjNN0)M*mT>#mjT}}DB#n%&AYWlhul8n7)8l6c*YoV&e*Bcrh1$A~ zsHQ`)afHubdSE^pB`COhvSTHrH|$$JuE2bAb^P_)d_7m>lE0}5ixqS~=xU3=J&O`m z{waNMEcS9x-QC$ImNUAGw8HPsXLmo=uA&4tr@76{WHdclKV@`;>u}lT5LJ89mZPMO zO68)=@opD7PMq+X4nhY45&bxb+479YwaG*^#2T_>3B(EnVH}>1(90k8VC|t#^v<(C zC0!_Ia0~VapeTK3B2W%65PwZUhR^&qxl)!2m8q(#v6J?gxp}^YiV+Qeu-M>f=NEE4 z9QSPXcoO!+BHQEzHsx*2*7tm`-9=6SOLh~Kg3g~tX+WOO}#z%G?1&{fjaLFJ4FjoL>W zIJuS-E1QlxuU);ItDM`kIpb2WBFVQO`+Rjvu+>~z3wgH^D`ff6bO&dJo6!gU6){oD zF*jjmU?5GtbFUkConhxx$&p9v2OaU}1nD}Xmp;Kx-z@D+VEv@xH0(9ceu!*<-emVO zVLnR2f7~@er@&$65jCgoMJ%wd^Tr5;E;Rsw&(6Pe1K8%O_T1@}g7(7isb62lzHQf;x7c=W>S5eH46+_%BiCX*(0AsBn z83YjltgT_ewT#7H=R-C->iCc1NCs*9)Y!B?>>3iR0?S|rm}ywhPg@;9O8R8x(?c5i z#;UQ+`yn#}RcfCUZTBa`5~#xZLQQyCH#f#Kfn6155d$@0a;Go1BQ1j$R-weWI$qzL zw~SP&2DY-sX~_qkn_}!nG3c;XEbj)$tFk1vUZZ^$i8YuxO}&dNdmBlI;dJaNY3Utd zAu@XY*)8)}Ftld>jj3p1x_it1<_%*9*J8(8uUmh>=i#kg21~A;J7t9VCjzeGkt01$ z0qoZc856Vu1j_@X#?8+J^MO~vEObfte?uL0D{4?Xjo-;!>YRXVp-t{(+lGt8^Cn^LaW1Cx!KsySIm4 zRb3sW-CAFfzKne`*~pz;)jP>GOoLuo z&+~lObUNh{0_mW5v14hg@wK<*O03Wt$;=@{v-5p_yLS;o~b+B9X{Nbg63* zkAXZ4=h>+jmOXs)!c=9iX$|`gOF~3&gu|;IV~nTZ*y$Ayv{Sq`m!2^U?ANp}4K7Lt z%;Ac%C!MNR57a-^V8Jm6&;1b+n%L1UtSPdWlxT&%q6AIL-wz2k%_}4uSiqYq-ea~N zWjb0*$g+`|U9lCjLdr+W?;fCX9_N+{SLQB~y}V2)uHakIRlP|CXo-A9?$N=O|B_SQ zz80qVF`YY;_D%J3;kT-9E~>kM--e7nb7=LGM4eZSJ;0`AkM&!<-ITTM%nkR*MYw<| z_n*+cC;OhF@&oIn0nwS2#>xzR$Ngq9dPQ~ER$IVQC}JEh!V$lfpRm;Qc*NP;k0xcU zvb-0WHVJnCOzc;|aH7iW{4UQaK(n5T!`8gfobJyrYoADfXAAzKCgH`w=5BO|&Y0Fb55A8>%U35raC{%vzR_E^P%R}sZKbe$M~Xo6EC`8_3o`{~XYU1* zkahN3fkbrq^Mn0;hlNGuWGR-yEjJMp&@2B|FWom7AzJm%v^Z_B^dw6wHz?sdHANpD zgL8D)-+v}==}4(~*Tlga^bJ{t{Ua4^*o-GVN*g5T)oVG_!^J6JBq(;Gq zo4G(RI)`fUBNX^>`o(VHF#n&Vu+f`kU#dk1oe6Wx2 zMOT>|7hcPVuqR4~M$&38HLP2uN-?I5LY~*3uep6Re{I*m#xw8Me{B$kJw01E zFFMG|krJ8I6sFOSP+{-KX6Waz_ZL#K>o!TkKQ{C<_;!Ri31sE*+P3-`D7rag z(GCRL90Fd;NG!Zl*PtARq3|x)kuZH@D>#0t?SOHKDTA9Y5OCwWb;*Di#zWh=Yw~sV z$Be27buboC)cs0@TO!&U-C>1h+DR=OAw9!{lZ9@yWn?8{M+J%LRMkaHA{=jd6#YAx z`?;RKCy#{EPK(iu^Mg(98$QnJR8yq)ihM)~uQ=JpVctprYrmSooid{l2L3FumQxYT zL)^o2iJ`^g*A3@^(pb}zt4@|pCNHSm{E+`FsR=ZUqYV@Ugk|^{rC;`O;JpP-H%CtR zIUtCwqr2PR=rhFVsk*|Ow_@_GNUy|77kR%q$Eqd9u6<0T{>fV^Vy{k#Zyfnv(Z4+m zT03#KRF5Dt5o0l)$mwzcUY@VV%4}*Xsck%6G5L}n7vk;G-rj^PDzQ3T>71GMD*NM>*OE<}g zGLe^iIR8qg{X{by2`%q|b*f91{1fHa=v=;+xEkw6)nhMcO6JY-XHZCmm1Yqhs3m44 zjh5RyirG0T$2QK4#C@N$OqcS&qURX~noN=BRQ&An#*8P0q`}0FsvE${sv=2kkg@}D zg2*;>-DeMd4Y{Ai_QZjk(gwEI8FECx1JIs|0H3lxsW1AR;f^FUvh9ERw$!!H%$Dmqdd`V`_P6L-lbqog0lpXZ+$pjlc0xb#r_y z=4I$N_3vzRpK-PLAbxqhj~+L27jJGzI^Zy6f%lTrCsYq0ngj^^?k>glZso7)8BxA< za4^{J?Bo8l5;*f0z7f4-BjMplKa%{|Rw7?4`Jp!Us);%8?bn2w4*-8hZxJwet<7R^PsV~ggWGF#^$ zqctbwn75)tcJ?5z>`6e6tS7s+4Htjd96vYu(RT6Cjf5E|rRvu`|ML9f^E1^lmsgy(%^*UhcptRcLhB2veEgf6m~P@CU$Llm5T@ok45d~u+D9BTdFU0 zyxgA=#7^Zlez!sn9;-TZt#76d*95gAQz!Ye-zg2!+k2I9v??$4C*+1X@p~*HkLe3D zy(orCoHcy*`m%=d<;WB|4{qxF9y!pY<(lQ8rwW!YXbh8n8U=8A{+@qy_Zf}u7hSPF z=QQGFrSTu)jqOc3r6)n$z6p((b*YmI8SV;ZrvKevz4Ups%qAHm`Hm)N-rd^xHlcC>A}LlGoII0cj`} z|Dvx-kkQS@!7tx1p;=_mv|>_o?Y_Q|?=9x8P(zq$mmH-xx-zS+Tg*4m``($KP6CX3 zV>D*2C2UemaM7@k&DOnsnSb1vxUgln;5LeFHNQJ)OD`O~PRU~{!^aeT9HW1npdV@L zd~u_G@v`LbWy>d{yW`spo15DM&yDy{%}|*qIrgIf&>mOLTL;7<#ojtlnOM-_}DO>~u=3lgH|MvE3=~Mgr z9kAQ`9WXnYT5!MOx%G3ZkHj=T!EF@>iKT1aj)UkZ14D*9oN%XYAZa zt+t;u!b!cG$X_BFFtIe96hhLq)z-pEs|@GZV{iv&*4c6vtF zhm^~ZbzAfeIZ3`Wq^SN75C0x~XF`WZMCU4+4Td8KY%1w9wFz%iAOxTANcFTHAryY$ zM0a}P202qndJIa;Zyc)`Y3zUAc(`LojH)UmBsN1#Sm->p&336}4c0MWmTA=ku2C3~ z^khD*(E|YRRXr+)yxMWX4&>t-yfF^ro;6`~Ne#>BiQxts?AAO>s@)GpiLStkv z2!lB>S@Rt(ir420>waje#FX#Lv{nlY3u8Upn~Ch#J~`OjALiDiQ?fND7mh1v{ZaUS z6gO9_pXH+lkzuh|>R|K-eU#(sY*!@DmQ(VJJF_OwR=NK9u7cn?3bue?GBWY$3H0>s zS4U!8zbh}-tJ}3KdS9XXaOmeX>)5L2hhH_z@_V(&KCxS4iQ2a-NPQF*wny3KeErl$ z)|O>>rc|r8>gkCkBY3Y zu!FdC>9oh1TZ52ludy*2Oq3lzb|>Zw)`tn?#wNWkO30TY@T$wPMH$q(8PXY1u< zL)6ClCB~9JCVshbBF|TGrLIGf)-l5OOGtiLt=AvI_*0Eu#)d0ej`3$_R4wlh%>=%E zDilxmR^_Or%ja$U%7W`o2l}o29&O{Pes%eA-xU+OZmIGf$g#RSRqcq;qSLVph*3O> zdGjHbWpA@GRoKb7WEh)f9jR3U(q^~T&2+bf@rhLu*#IcL%?M6S>0E(J z6!|lgjgN=zCuY#@z7+etEMAi^E;iXs)>d#_DPVir^bJ~`v`lqcRyM_p<53c8pX$^i zB3&_^8>{}(=;V(7hqveh<>2a7 zVw17t?RQV*zx)iC?1*_!5V3+uxlq!PrfZd#h)i*{*zP5VO{=Pl8)XtZ9T%pGlGFCLAS=v*yV$LSf5S*XLD!am!fPE{uk-{Ra z)erc6_N{zhO)$n?WRyHaPO8&ht1aLRB;?C$zKd#}w^4&jDXrCV>!qhwX+fHLKH#`D z-cDQUb)9S&ktd`+!`qF@DWm;DQX$8+$$2qkZRNMEz~9fb|MOXIvcHUZ)(#eq|Mu|s zABT-S#fIri;d_fmaHIX#fG{tJojdf8v)%XSyZ`CbJZFp~W(N3<`K-ySV7owt@T+jZL`%KC!WOM?s3^$yZ6{0 z2s9Nt3AT^}*d&QbP-gprKYqwT%CX$+Q?^q+=|RMFT@gG(>ebvsT5}d7$;h5H9O;Sc zOW*S+OFHipeVE|yZZKoY%T9nPZk~00UZDmej|m~FGDKvQ{dv;a*w=gBzS9_bc!2)r zl|(QKolXKyfFj&z{*nYJ%oXBd?c`?d=GN3<~AvHwQvsU<*NjAjHBPhWW4AqW;&T za06TZUCQqD#%}HUjtoApdn5fzR0UZ{ng4Z(zmNrd?!piM4*|ao{tp!j@*}jX!yjP` zx4*u^0daJ&u(tfwgZKy9ar_@*qkiU7S;gR5Vd{}iGLzQgux{_#?+)sH({!fMNg9;% zVYBeO!Z639Q*@V~bAlS#QgSRf$AX4;4@#HP6(*xj$m>)dL3!=S{Rg}MdMxBmzYjqi0!|PX@4!8UjzTGVEH>RCEcH|@BFoZ{)Yt+&+p?chF{+K z_^S&%QV-p=EZvGnRN$?)dz1bp5pY<3a{x?801N|I00A&CTpSQV=)FQ9LcD@7A)pY* zLP!W={+pw$d+I&AfZIO_%nMOhds!Xcep{-(i_|u&Q&g=A&6P~DNF`U1o_j*S$Xqg*(#|u=60&N&TbaU*H`IWw2NSNgn1Y~9-L@q4;_O!fvR zVg*G`YTPfVZtPw@4b9igNT@OFUPmc?p;wA)_yO1phv-d3etDP-n543vT7J^y)Hnep zY=E*xk=Il@jCqIcIlcYL(q&;C=$5KRYCu_!_Zvr2ga_8-eKqyTDqe@H9=6A6r33@1 z{^COv)`SMqTTp*m@7;4Ao)$V9sbROmki&7^D%pk#!)m23CVOEW^_S7?)SMrZerq=X@S`%j1ocf9J z39OV{gR^CW)NEMj@nVvVG`I3j3Dft?z z>J-jobFPsXXTB?>16c_r*N=X{RJ#;}B7ZBSr8D9nCR}mQpb8XY#EU1IL4A4JMX?MH z!o4i|30cqMLVc1`#KGitEj3-u72IB4RYEy0kzcFgeRXJSl6E=bkoAK}m(w@rg|A<7 zR+e9;Y`iti*RZhir<(J%KCjZY-;%uLSnZ@D+|b}oIVYwrBuM8?j8xg9oL%c85ZsOG z9b~%I=>O{WYBY9sg$vwXeLQTveyI<&Qy?~Q>)quh3K22=BTD>d`rwED2l{Aj{Kl6B z-#56oU+V)7%f|z;;1}eF0YUuts|ZjCY7PbP3i9%T!60*(g*i|VAjBv1Ykk191|8t~ z*fTKWeNvYEOel0`hm;(5ob-LvN7YzV=CTr5R;}#58ML3@P5NcCfn}!vS z)271GsN9CL^#*k_xlZ=R-^v*5=9VhH`H^d5CT?xmmnqTH1zVhf)m^!ua;_ZD-@ zs~#e%&JR@S8NxuEuWocRvxF}T@tp)AyAh%`WHF@t+`%K(4*LF+RP>3VV<7h_CS{*O zC!F+2)Er&JW-KT&JA^COCY%LhWRhrEKYlgha~uhciHJJ2VZ2Xjo^cF^WOk>qo~$`p zhWA>RXw1q=UifzsbP2!mIqQMK?;pISsY_(?xR$511lCVFdqtYxV&|256Th zTXBHqZB490UQ=42mlDeH2^Y)2nh&41o(ge3tkZX77~vD5F`lY;o$}F^MZa; z!Ap47e{cUMG0D(rcUX|a-hbW_IHQf^#q2%7(k``>8vwM(5HB_EQ#0e|rWwWhSrt|U_J(5SK0nKneb zG#ISN+&J{2y8iSD1y*AR^EmAiGnT5pKB3&L6k-`vPVI=mGSKp{Ok6xn(Sm}Hi)f1h z;6!P{+3r3O)k#t`+BA7xR_iHZf@?xPz8co|0&lUFpO8mRNGhSmH?gPBFapDom7O?1 zw?8D26_rSjop% zN`cp7CY*3LG$zWJI$s^rSH6xyqhd7j^i?rlYj=NzM-v$L{zYTXe5w6U zG?uH@1(gDht2!~-xEQr+g*BLiQ2zNmK;T=Q9qz#<((t%#Ab-GA@M7>*p+1_8iL0v!P+0={(H@b|EZmT{J+x?1dkBE{6~d*miNrk8T8;( z+}p3|fWz_&00Dd!P<{&wAfLIpIRs+C$14C8L zD8L;4rY!`aLhwKg{cTG3eEvnh<$tCGwBX@IKtg3iM%E5xd$&K(VdAh9VcAMRw0ZGj zBC`=spe-LqmAvZ&z}Tub%WxYj8y=;pnxl7;aE6ys^NPL`ZOncO3)Flge(<9TSU`Vl ze`0=Pfv_RQO!&w2{xe7`A4Kj;8v~PuS1N!o;_2fWo6UP5^F5&Rb6eX`mkhmNDQK*(txadTPm5bH zr2V8(E5hrdzXrnclh777zsBsDP7(sb=R0Pr|CUD(-`|Wc_dJ%_%X7uRIlQ-D^9YB9 z0(fBX?gK9f#0TYpfWdGrK|v4!AwB_khXP)EfxzZa@bArte<`Sp7-e|h82^dMCUG?x zQ&yqs6B*CUwi_T{Z6!{Cu+-TpY23=Om{*gIaz7HwTn5L`O0B?y2aIQ#RB54Q_6zud zIXq5|B~eArh|L{uqTK^_6PVSV2Jw74ov+6}hMqJ(%fu^6CQA8IQy2BcX2$lgW^V7C z__kb+jDdyqEOC=U@(0@!*7q&c?h=pwAo#v511I2rKfmtw04#nd_`czC{eK%Se}=8Q z=#MI{92jhFG7ngnfGR#Lp_3tU`JT9sFJ@-`$SoopAjSiU_4K+KWE3dUN!O@U#F=kNIQAK&%+| z>SzV(Q4SJfvouynLO0FxH*-{Tt~8`o2_YIb=piIix8Hbwpk3h0^k1lyEA_o1{m7v1 zjTddQlUgQDvp`;lW8)-s91@mNfGkx_FW4bDcACnY)Dczf4?5h9SBrH)YSvW-Wm!&i zbL6hUPhzUM_MNX=sW+Y+AB#)J&Q7p>Fj+xGl=Tv_jHVsco4Ev3k-8{cC_fVWdO1EMtl@Ih&rk;H zliOZpW8)-ph%*hHqu0A@pRA6T15dwk1l;zS}oFckCIje@9(qwb?)05Xr+28W3Hw|af=k36+ z859=iwx++X!;DEy%Jqu^{0?#+IpYF;92UH0u}-*7nVTr|5Ys6qA{mE%{qn<$8m1V( zme<&WbE@$WjCtH6?QERpsKq7;n9Kg^ryyK7qLEU!&p+J0Np#gUJdJr+RBl#r{#htB z3!=}T{Md-l;81A4rnOHL*8j=g~W5RQq@1S=yyu*+H zFY$cdG_j1-aASHZL8?`=9MsISJ{5|Ivcw+a99SX-KoLBV+2^7&P-i!Nj>4c}U`A*? z;DsxX8Yyqo%O9)UZ z+8GXcZ@-SMa9Cb`a{)nfFb~gtb_}!Ng=c?2Fg!&B@xU`FD2NZt2e1I~{T5sQO>zvZ zg8ql;AR_yeC&XAP`9-3LnaWH5-;xOUN4xcJkvK@-&!Ye* z@ZNq+A{-V3fq~%JCWMy{0^@~Q0Qd#?EnpysITYS@3fB!V}kjpGZ|m zx*44zsX4aBiL`L2T8I10H|e)hAk1Qrqx`qr@PL0`y#E$AqRM(9CU9Qv?bqDEVJ*M_ zFc1iXLE!t95Pl&rgjWCrfC%#N3qbe*->%rm5JwNNgPJAr%#3mx)NC7;J16_ZW^d7j1IUEcT>u&ny)Y zi(Kx?jVDT1sB*dw5Yz_f#{OHvp#PsD@5lYia`o7S;=Bk>*1i3jFgPp#Y60K{13?g8 z5TBqRKM1Z)06d$7H;?!P_<=w^ehYqhZux6x%7|69L6XFmGr3^1p~JXY7JDe^nfdT4 z^#h=8j>4KfyMA8WmM4wEK|2PCX3*JEu$FRZeDPQSrX-uI?8-W{ z5i0FRV)M)~MA^eVBN@L@zwGt9cs`|Q?e4eE0zclpgSCNy_GCXCEPXo8Pjh?o4*Z1c zb{}!xU>SSNzG3S|pc6o~)%)b9^SoTZ+XB2bh_BJe&ht3+vB)nH}-%rnu=s~7{@Yf; zJsUS}pJ)D_cHyuvJ^%m$@7=?D_z)o;3!tDmuLZo62EcgXag7(u%g4tL*Ye+C`*Y#7w13FE_Nhs>VHcZAMAJc`4?v|{tWwslQrz`X%`Or z_YQ6RM%cf2xEEqZtg?~}FFpdi6l^AATFl_a{yI+7a5$})zSX4=4p>@hXKW}53>+7d zTYf@)c>aMTwNxcvNQmIwtNo-WsPojn-*=gUS8>+>wR7KecjK! z-tX%!HEM4k3%};wBF40XfJk{HDd6f@cYH#H)Z}vI=intn(s2^;k1tF61v*#pBi587 zwpt058iAg8a#}%KUeWk3Q3OVgz_5}*FhXF-`nEtxuo<}T90iWvF%iAYExe!-&~=8) z#nQyY1mCpqzwBk7b72(=B$p2BC{Kim2=J|%S)Gf&8tC`LnS_8~j+EHaVh-KtJYY^e zH#>WFE>deAHmA(H0+Wj@nIYTRiZAIy257c*8?M| z<`+CoQb*U?@~#it?3*K3eLPGbKXQiN(0J2SO@ zG#sAP;f}tLGgVaf?65y~562n{5AJmjXw{fA(|G?Q@*}$S z)8L73I~!MX4+c3cH1Zv^y(W6$H@6qsbGM=;OT}UNfqUt{hh7J_hBNExG7Cpkt5k=N zy*5jU;z{#4;`1Wrk>xKZ$3D41V#fPX2UKW+OAM^OIl5BWKKL=?MbMskHlg^{HruPyv<07U5t45BW1sAJ{G=AMY{07~bS@^dD!G4kSr;2or_x)N} zDIRQUi@x1Z^gd`lK_{^^*7rX5WA8e*h0Cq48dDzZcvBBAxwOUQG74&qqyawbupef$ zoDq3-RaGTzV}(;n+NxS-{w50SQ;9NT+dFz#m@~UMGMhQN0J=LjW;;hSBz6_*p&j~k zv~%1fRzV!sZRK~A)nG<9;QIoQNlY1~vx#rL;%Osbg8uv>LA)9=?`?oYS??eSfbgmqA zrv2bl)M@mn|CRKNgn`sG)q1m1Z@m)U{V5qAO_J`!9J##b1HY&wT~C3BXvT52_f6f0 zzuZl&`Yio3m(R_2g89o*b^?pFMd5AVV@Jf2!a4I8#i-r*y{2jy>N-!BSC<)obsTzj zGO+BVM1dCVtS$dsbfji*(lKhe1qJUTXDvRICfpBxJ{?e8SR^}6uH18ihcV_n#iOln z3~KYE|Hi;u3r;hG_cTNwAaV%RDl6I?+Omp73WTtYzJED!vW>zF{Que6QYd@lZJfhP z4i1j)4rUwEcrUR0hW(%n=&j^It%JR7sQtAkz{j_7?*dU38_~a005Ni6h{-bzn45?I zdFZwhWMHHAmadmz)_DZPcb?+wX}E3GQ9=!>?Xl3H`#60 zSyUqmcJKMBd7cwLA(GT8^=5UGN+QeMdg49p`TeKJS5=rMn7Qvy$`;K_`r}=zvwO;a zBp^=4yFaib_wJ0j*XyUFw0$x!oKl1cX)gOlYyYy>5oAcqqtsKlw=2|O(%zq@+>XCW zrl3wkzk;2|dyeq@Tec;aeo?|MQ~92Beukvh18#+cT+hQ#7?w!~PjC?aCl}OdG?-2G z({ey(sd1j3R8_QjpQSeO_l(&-l@dEKzhE{uQL?T}1&g+e?t48d;#vx5S}wBs3uoQp zt7@#RdTYeiWwT7jMDEl~WG-3l(A!o~bZWxRz}qIVRm#MkJ#Hb|w(_({JKaKq>BXoJ zcEM3XYyL~`WP6$W%QZ>qauYjNml8Cog8b@tidwK{f7p|rsa;=TnY(w$PR-qT?fOa< z^};|{{ZRKI(naG)M}A#WqujM0wz_N@8ASa)&KV==&wlUzEfGS?RyMi3^NZT68+v01G>=~67d>LglT=@Y#gma4mKZ{PdyIucczfC+q@ z1vSL@H+F9ifc(cWZV^KaW;C$FPfiRNVkzkAfFlhtz$jvf@eAWqIeEgTmqNrLe)o4KblGP`4H`t6g)%;vC>YABr^x77I{ZpvX{Eop zF;_*=%#Lq&0?le<`V~E+cTav7OYY^F$o4*xM@{F$QP(y3xLEXPrpTg3BF_iYu_u(z zWHaw1m|gaF$sQ(L)Zo7BD*KxAo^drz6vyEL69%5E-)SVJz8bF6J}j@JkraPtPQRkx z6evT)w7~YY>$-+_ghF-F_(`2_Ek~cVMhROmjeTNFBY#QNx>z;hG}}f^sUhCn<<@yh zYdFA`B8$D>uP#Ex)2i{eC&gspv*O-^Pts>bRv9Q*>8C4x@%5Q_Q<~L!`t6BVv=aZ# zQWC>hbFWkUWpKPq_hb4N&Os$EiXYJy6j7Gh6L)xG4VIbuEeWi}1?~qLwn#szhcQ0S zU7XGB;;iZ6x@$_WoEju>=y?KON{Z`PTn>{}r#b=SLGu@EBx2SDUv4UwTuq$`l=L>I zFJ*EHPo|8owaIN0;>ZcLO9=1ym}=g!Ge9xYau<0CdtsbBqmNx6|6Trv$z4~&tG*ji zE4=p@3$LDaPO!NWr{ev^+w<2x!B_RPbc0&%^$q@+J7r$HICl8g%(+afip8fFhrAg} zE0YTh$QIT=y2O@!b$B#r{zQCP?)~y?QnBQ^uqlbU-Q(V&v;)1nX`P1i>RTKn!R4buowTe!XX*M4tp0eYiA1{xd?oP zI@*@7Ddi8@`_Ht z?t&jx1p-R1t=huu(%9c{EOZnLcBOGQ$LPuoN=~LPgwAVo=k}xxc&zU|)TJLsqts`T zTJ!KH=PKLe)12>}ej%x?H}uKuzH#z=-E%=k|IbODltd~zvBOi} zc#?GzM|`RI52RObO$qc6ASj$c@kBP01QHkfo5^-~>e@<0oS8^!9T<*l5qnQg8ga$K ze1VVfIq$>T`n|VEoiy3x((teBZy-8f7d!wkI1Osf2WGK0Ov{Ws4-Mr#4r6d~zdx6G zT3R;sbwLJ!l7ZV~$lx6z(chf4YgN_iV>%M5-I8Bfm;!!UwCrn5*sW0V>W^M#WF>56 zg6MF3;I4U=z0#Bs=K2eE?nnnopd)R*!9u43+PqEcMjNaOMnv?5K+^wiRH(Kv`-h;({nKh;)vFY8&2(d4HXXC7SfnH`>PLbeH zVzXM*3ujf;hGW5^51vN9D?SvhChXQ%d!bvu>A3l|>@+J<1J}o0m3^!E7tW{Y`lXdQ zib`aCV81V_|I+!4IB9KQfE^=YXEu$@-TTv3lZW^?t=4uuez$uV=Yooi>Ge@R|0)Mdz? z>%Y@WD?!_~A`@ABiBXbCN%~PoV6z&tLXDv6lF?@H+Hvd$$^k4Er#c~eHlmc!sRi}{oWs->NZS10Vl_ZjyPt2H<1k2pL zsbU`{sn7XD*2AK8k7Jqkjzz{ZappjI@3DKbT(S8dnUf=L-W)eM#i5!t|H0z!+h`K< zW0My5=$1ULD4W6PlS452r0QPY(8^`@u?MMQbt=823|lBNh-6?B&D3Yf%S6I)@V+Ar!G>t6e_E- zZ`$|L@jIqU{e!P{3cNW|+f5V7L*_5NA^mQ0IlaqjY<^{a9H~q*>@yatp;9Of+`}IN zHbpompwQn0#PC&vUAPaPCIb?X>QgLjHi1GRcef+Gry~h1aaVV5y(#;-HsGBLcT4m7 zpf!!krzg+5m=_hga%Ich2tG>T?d7F7`kvmJi(`QERpP09$~z|8O=Q@fAnkkdw0DCWm>rxjQh@Oue>vxCZA*pjGT%yx);vI=%x^{ zu;)Nohp?e_oW~Mh&g84f)iwGmhV*z1ZrP0K6_n^=Iqp7dI>ec&J&;nQNwSGOy#zKAN(+#0n zx$Axko_WHh*Ip4A6YM^sGq|85G;mmQ#Yo6hzTj>P>5jGJS$>gcWOOA>JcYNJY3n{q z_u)TT(zKF%6ZK^2cgs<>iUyioDuX#q4$E^s9e1p$oA-~dCht8oHReJ-Pd^&hlJ(rY`z00F7o=Sm!-8ea_F8`T1(M-CSqr1(W z8F%$_=}D!JrdmDKj)ZRBtBa|uPxDFA@1hVJ={xe_$srnrQpyagKTjDg$KI_3J~g}Y z$+dFlTF8KxQa@h!&ynPMlh}JSJVdu$zJCq!=NhdU^bg2AL%UOAoppYCTDR}=_cgi7 z_HMTmQVHeWZf#Rp6ddJJ^glLf=thIQs^mzX4J!8hy-h!vzNW5qvOA>3 zsr@3)oAV<-E?x2^;t4s`ZHb^F-kOJQIF@ z{=(!Vy=E)wB&I9Q?)q{ zDNFZ-eQ9%~dh`c{w1A3lH=G_Uw8!8No{mv`#J2itw4m%w_)AA`+grOnPm(kH)^_~L zVl8N^J4tf=mGt85z=TQF!oJSowu^f-@BO~{Y1o-M#Z3C$Aa}pKxa#lAcg{$v<@JZF z2AomKR}7FpkuX6ySIwbH+v?PiYtoq1ZINvrJJT2SrnNu&K1G0yIC+V{TQO0NAkl7r z2Db%Q+jT&ix&xny0C7r#x_u`H)PaaURCp7zu-{M~wIW13YPD^dwPh0Z$eJ2t@(ilU zjR&p(yRc(x2)n52WO|U@D71Ew9=ien(1<6I+X5Bf00>|NEx_i(S5Vdw%MqA$@UvHl zmIjuicoTl&z(1gthBphs`c^@EQ1L6=Ai-}vAww1e>mp2$XkLE84G#PO5HcJ!aE^ls z4&pf=++e}4At1y0kYR>J_cj7R1$dI3b@+6pA z7EOVE+~B|^!I9w_fw!HY;LsKb#|;!*9~&8z9~eYXprC5mxPgJo4I_i817d6_FqGoM zxWR#I6C%S!9Kr?%u2hH{9=N<7GM+RGBp#+>d$_@ZOOPSMMzBM`Vk$_68z{I?4>D*U zP%{8iSE7{dfr3IzmZ%fNe$WKb!HW)RLjVqCd~1FH*UH$6BiZlpag4_QR?ff;16+y% z8RHlqIt)-@3MdF%+NcUG#Kx@>cH`TbEC~Pc@L;QzJ^bkWN2-jVqYXg7?)e~A8Stxd zP`wG{CbGr`v3bXY9SAj<7(mH1i5Y04zy$Wzf+2v*OCX1GN&sDJA+AHAY;R?&2&kq6 zvh6gqwl^zGKv_p55$Y21AgFCr1kriEDei%H@;@#m4}t7IhP1y?Dgt{Ds2vOjS%L(? z;j4jYsqLb8JutDHjP6BWQ?fLE2#sWH4 z7ZIpf;pG5uixpf906Er1Y!JAX2KZlN9TJ0z6|O_aEmp7!9XVD>fYP^p;@li$|DZ|# zUt=AUgmMI~HpU+7cC9gTta4IVAaGS41CF3k9V72CQCSg;pfAsL87N2KDoy|bMvs6P zuiG`8$gy@|gTPhM2{?k5o4Yk$p=aw*;B*0Vg25%400bH`5o&XS1)9jQ4q=19Rj!G> z(P6QIWtGT|6w6_W6<3iZZY=_wDE$2cPOw;cC`aIONbIq~au_UzM2>Y58w9S>NZcIx z8z}G#IM#2cpkjqf7IA|B3l@=MJ)wXF0$2GWc1O1B1vedmEq!H9LpcIh4+0Qa=2sX5 zSUb200h{P!u|eRfB*gB>cKyz#Bd}N#&qBotSMvZ6SiVi$wLF^;uvq7?LEx(F!OaoO zIoz)T6)RkHgBt``a)TVJs45l+Tm?9|If6OXKah^VMKriUfF(4@v1+S9#rh9|8r&Sg z94jwSix2bU2iL#g1_4&TAjf(O8w9Re80?NfPxNUTP_e@0CIAGMi5><47Mmc)ihm9Z z1g_E(+#JE&uO*OrY5W;5rZh0*B)kunGh@ zR#`2mSpPv2g543+BZsZy7(BdWKPzo?jv&h}00fMO4*P&cB*ufKFj#c487d5+0xg1# z16+cETUUV@`v~K}wGv>w7bsq`J!eq)``A%XcQ4=rP!(NtC>ykWz&OTz%U1h-q5WVe- z6AtbtmD?%fn~4i+x8e&ZQT^r7!B67Ql2Etyh^Y6^;7eWNicHPt7CgK+wy^*GA0~6SX#fBK literal 0 HcmV?d00001 diff --git a/vulnerabilities/tests/test_mozilla.py b/vulnerabilities/tests/test_mozilla.py new file mode 100644 index 000000000..9db6e07d4 --- /dev/null +++ b/vulnerabilities/tests/test_mozilla.py @@ -0,0 +1,98 @@ +import shutil +import os +import tempfile +from unittest.mock import patch +import zipfile + +from django.test import TestCase + +from vulnerabilities import models +from vulnerabilities.import_runner import ImportRunner +from vulnerabilities.importers.npm import categorize_versions + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, "test_data/") + + +@patch("vulnerabilities.importers.MozillaDataSource._update_from_remote") +class MozillaImportTest(TestCase): + + tempdir = None + + @classmethod + def setUpClass(cls) -> None: + cls.tempdir = tempfile.mkdtemp() + zip_path = os.path.join(TEST_DATA, "mozilla.zip") + + with zipfile.ZipFile(zip_path, "r") as zip_ref: + zip_ref.extractall(cls.tempdir) + + cls.importer = models.Importer.objects.create( + name="mozilla_unittests", + license="", + last_run=None, + data_source="MozillaDataSource", + data_source_cfg={ + "repository_url": "https://example.git", + "working_directory": os.path.join(cls.tempdir, "mozilla_test"), + "create_working_directory": False, + "remove_working_directory": False, + }, + ) + + @classmethod + def tearDownClass(cls) -> None: + # Make sure no requests for unexpected package names have been made during the tests. + shutil.rmtree(cls.tempdir) + + def test_import(self, _): + runner = ImportRunner(self.importer, 100) + + # Remove if we don't need set_api in MozillaDataSource + # with patch("vulnerabilities.importers.MozillaDataSource.versions", new=MOCK_VERSION_API): + # with patch("vulnerabilities.importers.MozillaDataSource.set_api"): + # runner.run() + runner.run() + # pdb.set_trace() + + assert models.Vulnerability.objects.count() == 9 + assert models.VulnerabilityReference.objects.count() == 9 + assert models.VulnerabilitySeverity.objects.count() == 9 + assert models.PackageRelatedVulnerability.objects.filter(is_vulnerable=False).count() == 16 + + assert models.Package.objects.count() == 12 + + self.assert_for_package("Firefox ESR", "mfsa2021-06", "78.7.1") + self.assert_for_package("Firefox ESR", "mfsa2021-04", "78.7", "CVE-2021-23953") + self.assert_for_package("Firefox for Android", "mfsa2021-01", "84.1.3", "CVE-2020-16044") + self.assert_for_package("Thunderbird", "mfsa2014-30", "24.4") + self.assert_for_package("Thunderbird", "mfsa2014-30", "24.4") + self.assert_for_package("Mozilla Suite", "mfsa2005-29", "1.7.6") + + def assert_for_package( + self, + package_name, + mfsa_id, + resolved_version, + vulnerability_id=None, + impacted_version=None, + ): + vuln = None + + pkg = models.Package.objects.get(name=package_name, version=resolved_version) + vuln = pkg.vulnerabilities.first() + + if vulnerability_id: + assert vuln.vulnerability_id == vulnerability_id + + ref_url = f"https://www.mozilla.org/en-US/security/advisories/{mfsa_id}" + assert models.VulnerabilityReference.objects.get(url=ref_url, vulnerability=vuln) + + assert models.PackageRelatedVulnerability.objects.filter( + package=pkg, vulnerability=vuln, is_vulnerable=False + ) + + +def test_categorize_versions_ranges(): + # Populate if impacted version is filled + pass From 59b420a0c62363bf818669d6349cc46f68e98d54 Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Sun, 21 Mar 2021 05:50:13 +0530 Subject: [PATCH 05/16] Mention mozilla importer Signed-off-by: Hritik Vijay --- SOURCES.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/SOURCES.rst b/SOURCES.rst index 281f77182..1c8134e8c 100644 --- a/SOURCES.rst +++ b/SOURCES.rst @@ -47,3 +47,5 @@ +----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+ |suse_scores | https://ftp.suse.com/pub/projects/security/yaml/suse-cvss-scores.yaml |vulnerability severity scores by SUSE | +----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+ +|mozilla | https://github.com/mozilla/foundation-security-advisories |mozilla | ++----------------+------------------------------------------------------------------------------------------------------+----------------------------------------------------+ From d1f8315ff2d206c228b9150e81e63620b86097f4 Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Fri, 26 Mar 2021 03:54:47 +0530 Subject: [PATCH 06/16] Add split_markdown_front_matter() Signed-off-by: Hritik Vijay --- vulnerabilities/helpers.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/vulnerabilities/helpers.py b/vulnerabilities/helpers.py index 77376ee94..c8dea9ef3 100644 --- a/vulnerabilities/helpers.py +++ b/vulnerabilities/helpers.py @@ -22,6 +22,7 @@ import json import re +from typing import Iterable, List, Tuple import requests import toml @@ -79,6 +80,43 @@ def create_etag(data_src, url, etag_key): return True +def split_markdown_front_matter(lines: Iterable) -> Tuple[str, str]: + """ + This function splits lines into markdown front matter and the markdown body + and returns list of lines for both + NOTE: lines is expected to be an iterable containing strings + + for example : + lines = + --- + title: ISTIO-SECURITY-2019-001 + description: Incorrect access control. + cves: [CVE-2019-12243] + --- + # Markdown starts here + + get_markdown_front_matter(lines) would return + ['title: ISTIO-SECURITY-2019-001','description: Incorrect access control.' + ,'cves: [CVE-2019-12243]'], + ["# Markdown starts here"] + """ + + fmlines = [] + mdlines = [] + splitter = mdlines + + for index, line in enumerate(lines): + line = line.strip() + if index == 0 and line.startswith("---"): + splitter = fmlines + elif line.startswith("---"): + splitter = mdlines + else: + splitter.append(line) + + return "\n".join(fmlines), "\n".join(mdlines) + + is_cve = re.compile(r"CVE-\d+-\d+", re.IGNORECASE).match From 7aad267bed1317927cf262d4edf97977b1b67c65 Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Fri, 26 Mar 2021 04:04:37 +0530 Subject: [PATCH 07/16] Refactor according to first review Use batch_advisories for now. It has it's own problems and there's #338 for that. The generator thing won't do much, since we are importing like 10-20 MBs of data. The codebase already has overuse of methods starting with _ , I'd say avoid them. They don't help with readability nor are they trivial in this case _parse_md and _parse_yml: The name is misleading. The function is parsing + enriching the data. converted to get_advisories_from_yml and get_advisories_from_md Remove `"branch": None` in importer_yielder Group imports Signed-off-by: Hritik Vijay --- vulnerabilities/importer_yielder.py | 1 - vulnerabilities/importers/mozilla.py | 100 +++++++-------------------- 2 files changed, 26 insertions(+), 75 deletions(-) diff --git a/vulnerabilities/importer_yielder.py b/vulnerabilities/importer_yielder.py index dfe834224..0dd964ac7 100644 --- a/vulnerabilities/importer_yielder.py +++ b/vulnerabilities/importer_yielder.py @@ -240,7 +240,6 @@ "last_run": None, "data_source": "MozillaDataSource", "data_source_cfg": { - "branch": None, "repository_url": "https://github.com/mozilla/foundation-security-advisories", }, }, diff --git a/vulnerabilities/importers/mozilla.py b/vulnerabilities/importers/mozilla.py index 2f4662283..a4dca2149 100644 --- a/vulnerabilities/importers/mozilla.py +++ b/vulnerabilities/importers/mozilla.py @@ -1,21 +1,19 @@ -from typing import Set, List, Generator - import re -import asyncio -from bs4 import BeautifulSoup -from packageurl import PackageURL -import requests +from typing import List +from typing import Set import yaml +from bs4 import BeautifulSoup from markdown import markdown +from packageurl import PackageURL from vulnerabilities.data_source import Advisory from vulnerabilities.data_source import GitDataSource from vulnerabilities.data_source import Reference from vulnerabilities.data_source import VulnerabilitySeverity -from vulnerabilities.severity_systems import scoring_systems -from vulnerabilities.package_managers import GitHubTagsAPI from vulnerabilities.helpers import is_cve +from vulnerabilities.helpers import split_markdown_front_matter +from vulnerabilities.severity_systems import scoring_systems REPOSITORY = "mozilla/foundation-security-advisories" @@ -31,61 +29,40 @@ def __enter__(self): recursive=True, subdir="announce" ) - # Do we need this ? - # self.version_api = GitHubTagsAPI() - # self.set_api() - - def set_api(self): - repository = "/".join(self.config.repository_url.split("/")[-2:]) - asyncio.run(self.version_api.load_api([repository])) - - def added_advisories(self) -> Set[Advisory]: - return self._load_advisories(self._added_files) - def updated_advisories(self) -> Set[Advisory]: - return self._load_advisories(self._updated_files) - - def _load_advisories(self, files) -> Set[Advisory]: - """ - Yields list of advisories of batch size - """ + files = self._updated_files.union(self._added_files) files = [ f for f in files if f.endswith(".md") or f.endswith(".yml") ] # skip irrelevant files advisories = [] for path in files: - for advisory in self._to_advisories(path): - advisories.append(advisory) - if len(advisories) >= self.batch_size: - yield advisories - advisories = [] + advisories.extend(self.to_advisories(path)) - # In case batch size is too high - yield advisories + return self.batch_advisories(advisories) - def _to_advisories(self, path: str) -> List[Advisory]: + def to_advisories(self, path: str) -> List[Advisory]: """ Convert a file to corresponding advisories. This calls proper method to handle yml/md files. """ - mfsa_id = self._mfsa_id_from_filename(path) + mfsa_id = self.mfsa_id_from_filename(path) with open(path) as lines: if path.endswith(".md"): - return self._parse_md(mfsa_id, lines) - elif path.endswith(".yml"): - return self._parse_yml(mfsa_id, lines) + return self.get_advisories_from_md(mfsa_id, lines) + if path.endswith(".yml"): + return self.get_advisories_from_yml(mfsa_id, lines) return [] - def _parse_yml(self, mfsa_id, lines) -> List[Advisory]: + def get_advisories_from_yml(self, mfsa_id, lines) -> List[Advisory]: advisories = [] data = yaml.safe_load(lines) data["mfsa_id"] = mfsa_id - fixed_package_urls = self._get_package_urls(data.get("fixed_in")) - references = self._get_references(data) + fixed_package_urls = self.get_package_urls(data.get("fixed_in")) + references = self.get_references(data) if not data.get("advisories"): return [] @@ -103,16 +80,15 @@ def _parse_yml(self, mfsa_id, lines) -> List[Advisory]: return advisories - def _parse_md(self, mfsa_id, lines) -> List[Advisory]: - yamltext, mdtext = self._parse_md_front_matter(lines) - + def get_advisories_from_md(self, mfsa_id, lines) -> List[Advisory]: + yamltext, mdtext = split_markdown_front_matter(lines) data = yaml.safe_load(yamltext) data["mfsa_id"] = mfsa_id - fixed_package_urls = self._get_package_urls(data.get("fixed_in")) - references = self._get_references(data) + fixed_package_urls = self.get_package_urls(data.get("fixed_in")) + references = self.get_references(data) - description = self._html_get_p_under_h3(markdown(mdtext), "description") + description = self.html_get_p_under_h3(markdown(mdtext), "description") # FIXME: add references from md ? They lack a proper reference id and are mostly bug reports @@ -126,7 +102,7 @@ def _parse_md(self, mfsa_id, lines) -> List[Advisory]: ) ] - def _html_get_p_under_h3(self, html, h3: str): + def html_get_p_under_h3(self, html, h3: str): soup = BeautifulSoup(html, features="lxml") h3tag = soup.find("h3", text=lambda txt: txt.lower() == h3) p = "" @@ -138,38 +114,14 @@ def _html_get_p_under_h3(self, html, h3: str): p += tag.get_text() return p - def _parse_md_front_matter(self, lines): - """ - Return the YAML and MD sections. - :param: lines iterator - :return: str YAML, str Markdown - """ - # fm_count: 0: init, 1: in YAML, 2: in Markdown - fm_count = 0 - yaml_lines = [] - md_lines = [] - for line in lines: - # first line we care about is FM start - if fm_count < 2 and line.strip() == "---": - fm_count += 1 - continue - - if fm_count == 1: - yaml_lines.append(line) - - if fm_count == 2: - md_lines.append(line) - - return "".join(yaml_lines), "".join(md_lines) - - def _mfsa_id_from_filename(self, filename): + def mfsa_id_from_filename(self, filename): match = MFSA_FILENAME_RE.search(filename) if match: return "mfsa" + match.group(1) return None - def _get_package_urls(self, pkgs: List[str]) -> List[PackageURL]: + def get_package_urls(self, pkgs: List[str]) -> List[PackageURL]: package_urls = [ PackageURL( type="mozilla", @@ -182,7 +134,7 @@ def _get_package_urls(self, pkgs: List[str]) -> List[PackageURL]: ] return package_urls - def _get_references(self, data: any) -> List[Reference]: + def get_references(self, data: any) -> List[Reference]: """ Returns a list of references Currently only considers the given mfsa as a reference From df9786d65794804844d4379708dec24464bff58f Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Fri, 26 Mar 2021 04:26:03 +0530 Subject: [PATCH 08/16] Extract CVE references from markdown data Signed-off-by: Hritik Vijay --- vulnerabilities/importers/mozilla.py | 17 +++++++++++------ vulnerabilities/tests/test_mozilla.py | 2 +- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/vulnerabilities/importers/mozilla.py b/vulnerabilities/importers/mozilla.py index a4dca2149..f1a988cef 100644 --- a/vulnerabilities/importers/mozilla.py +++ b/vulnerabilities/importers/mozilla.py @@ -62,7 +62,7 @@ def get_advisories_from_yml(self, mfsa_id, lines) -> List[Advisory]: data["mfsa_id"] = mfsa_id fixed_package_urls = self.get_package_urls(data.get("fixed_in")) - references = self.get_references(data) + references = self.get_yml_references(data) if not data.get("advisories"): return [] @@ -86,16 +86,21 @@ def get_advisories_from_md(self, mfsa_id, lines) -> List[Advisory]: data["mfsa_id"] = mfsa_id fixed_package_urls = self.get_package_urls(data.get("fixed_in")) - references = self.get_references(data) + references = self.get_yml_references(data) + cves = re.findall(r"CVE-\d+-\d+", yamltext + mdtext, re.IGNORECASE) + for cve in cves: + references.append( + Reference( + reference_id=cve, url=f"https://cve.mitre.org/cgi-bin/cvename.cgi?name={cve}" + ) + ) description = self.html_get_p_under_h3(markdown(mdtext), "description") - # FIXME: add references from md ? They lack a proper reference id and are mostly bug reports - return [ Advisory( summary=description, - vulnerability_id="", # FIXME: Scrape the entire page for CVE regex ? + vulnerability_id="", impacted_package_urls=[], resolved_package_urls=fixed_package_urls, references=references, @@ -134,7 +139,7 @@ def get_package_urls(self, pkgs: List[str]) -> List[PackageURL]: ] return package_urls - def get_references(self, data: any) -> List[Reference]: + def get_yml_references(self, data: any) -> List[Reference]: """ Returns a list of references Currently only considers the given mfsa as a reference diff --git a/vulnerabilities/tests/test_mozilla.py b/vulnerabilities/tests/test_mozilla.py index 9db6e07d4..5e20f83c3 100644 --- a/vulnerabilities/tests/test_mozilla.py +++ b/vulnerabilities/tests/test_mozilla.py @@ -56,7 +56,7 @@ def test_import(self, _): # pdb.set_trace() assert models.Vulnerability.objects.count() == 9 - assert models.VulnerabilityReference.objects.count() == 9 + assert models.VulnerabilityReference.objects.count() == 10 assert models.VulnerabilitySeverity.objects.count() == 9 assert models.PackageRelatedVulnerability.objects.filter(is_vulnerable=False).count() == 16 From d7ced8b3aa27d08b5171a57b842a767dfbc4c935 Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Fri, 26 Mar 2021 05:08:41 +0530 Subject: [PATCH 09/16] split_markdown_front_matter: Do not strip lines spaces are important, otherwise it would fail to produce a valid yaml front matter in case of https://raw.githubusercontent.com/mozilla/foundation-security-advisories/master/announce/2012/mfsa2012-85.md Anyway, line shouldn't be altered in a splitter. Signed-off-by: Hritik Vijay --- vulnerabilities/helpers.py | 12 +++++------- vulnerabilities/importers/mozilla.py | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/vulnerabilities/helpers.py b/vulnerabilities/helpers.py index c8dea9ef3..6cdd7fa7f 100644 --- a/vulnerabilities/helpers.py +++ b/vulnerabilities/helpers.py @@ -80,11 +80,10 @@ def create_etag(data_src, url, etag_key): return True -def split_markdown_front_matter(lines: Iterable) -> Tuple[str, str]: +def split_markdown_front_matter(lines: str) -> Tuple[str, str]: """ This function splits lines into markdown front matter and the markdown body and returns list of lines for both - NOTE: lines is expected to be an iterable containing strings for example : lines = @@ -95,7 +94,7 @@ def split_markdown_front_matter(lines: Iterable) -> Tuple[str, str]: --- # Markdown starts here - get_markdown_front_matter(lines) would return + split_markdown_front_matter(lines) would return ['title: ISTIO-SECURITY-2019-001','description: Incorrect access control.' ,'cves: [CVE-2019-12243]'], ["# Markdown starts here"] @@ -105,11 +104,10 @@ def split_markdown_front_matter(lines: Iterable) -> Tuple[str, str]: mdlines = [] splitter = mdlines - for index, line in enumerate(lines): - line = line.strip() - if index == 0 and line.startswith("---"): + for index, line in enumerate(lines.split("\n")): + if index == 0 and line.strip().startswith("---"): splitter = fmlines - elif line.startswith("---"): + elif line.strip().startswith("---"): splitter = mdlines else: splitter.append(line) diff --git a/vulnerabilities/importers/mozilla.py b/vulnerabilities/importers/mozilla.py index f1a988cef..4d45f9e78 100644 --- a/vulnerabilities/importers/mozilla.py +++ b/vulnerabilities/importers/mozilla.py @@ -81,7 +81,7 @@ def get_advisories_from_yml(self, mfsa_id, lines) -> List[Advisory]: return advisories def get_advisories_from_md(self, mfsa_id, lines) -> List[Advisory]: - yamltext, mdtext = split_markdown_front_matter(lines) + yamltext, mdtext = split_markdown_front_matter(lines.read()) data = yaml.safe_load(yamltext) data["mfsa_id"] = mfsa_id From 05d93b2e3a88b15bceb5675b7cb46569d3fc216b Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Sat, 27 Mar 2021 03:39:13 +0530 Subject: [PATCH 10/16] Use the canonical way of splitting name, version Mozilla website uses rsplit to extract the name and version so it should be better in any case. https://github.com/mozilla/bedrock/blob/765a60450235d810cf941676e4a29f012a9eaaba/bedrock/security/models.py#L29 Based on discussion here https://github.com/mozilla/foundation-security-advisories/issues/76 Signed-off-by: Hritik Vijay --- vulnerabilities/importers/mozilla.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/vulnerabilities/importers/mozilla.py b/vulnerabilities/importers/mozilla.py index 4d45f9e78..1bca201b5 100644 --- a/vulnerabilities/importers/mozilla.py +++ b/vulnerabilities/importers/mozilla.py @@ -130,10 +130,9 @@ def get_package_urls(self, pkgs: List[str]) -> List[PackageURL]: package_urls = [ PackageURL( type="mozilla", - # TODO: Improve after https://github.com/mozilla/foundation-security-advisories/issues/76#issuecomment-803082182 # pkg is of the form "Firefox ESR 1.21" or "Thunderbird 2.21" - name=" ".join(pkg.split(" ")[0:-1]), - version=pkg.split(" ")[-1], + name=pkg.rsplit(None, 1)[0], + version=pkg.rsplit(None, 1)[1], ) for pkg in pkgs ] From aed84bc54c9051f3df8d21c735ba88a8c8de86cb Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Tue, 30 Mar 2021 23:29:05 +0530 Subject: [PATCH 11/16] refactor: replace classmethod w/ func and more Refactors based on requested review https://github.com/nexB/vulnerablecode/pull/393#pullrequestreview-624078540 Signed-off-by: Hritik Vijay --- vulnerabilities/importers/mozilla.py | 233 ++++++++++++++------------- 1 file changed, 125 insertions(+), 108 deletions(-) diff --git a/vulnerabilities/importers/mozilla.py b/vulnerabilities/importers/mozilla.py index 1bca201b5..4b82a14bb 100644 --- a/vulnerabilities/importers/mozilla.py +++ b/vulnerabilities/importers/mozilla.py @@ -15,7 +15,6 @@ from vulnerabilities.helpers import split_markdown_front_matter from vulnerabilities.severity_systems import scoring_systems - REPOSITORY = "mozilla/foundation-security-advisories" MFSA_FILENAME_RE = re.compile(r"mfsa(\d{4}-\d{2,3})\.(md|yml)$") @@ -37,128 +36,146 @@ def updated_advisories(self) -> Set[Advisory]: advisories = [] for path in files: - advisories.extend(self.to_advisories(path)) + advisories.extend(to_advisories(path)) return self.batch_advisories(advisories) - def to_advisories(self, path: str) -> List[Advisory]: - """ - Convert a file to corresponding advisories. - This calls proper method to handle yml/md files. - """ - mfsa_id = self.mfsa_id_from_filename(path) - - with open(path) as lines: - if path.endswith(".md"): - return self.get_advisories_from_md(mfsa_id, lines) - if path.endswith(".yml"): - return self.get_advisories_from_yml(mfsa_id, lines) +def to_advisories(path: str) -> List[Advisory]: + """ + Convert a file to corresponding advisories. + This calls proper method to handle yml/md files. + """ + mfsa_id = mfsa_id_from_filename(path) + if not mfsa_id: return [] - def get_advisories_from_yml(self, mfsa_id, lines) -> List[Advisory]: - advisories = [] - data = yaml.safe_load(lines) - data["mfsa_id"] = mfsa_id - - fixed_package_urls = self.get_package_urls(data.get("fixed_in")) - references = self.get_yml_references(data) - - if not data.get("advisories"): - return [] - - for cve, advisory in data["advisories"].items(): - advisories.append( - Advisory( - summary=advisory.get("description"), - vulnerability_id=cve if is_cve(cve) else "", - impacted_package_urls=[], - resolved_package_urls=fixed_package_urls, - references=references, - ) - ) + with open(path) as lines: + if path.endswith(".md"): + return get_advisories_from_md(mfsa_id, lines) + if path.endswith(".yml"): + return get_advisories_from_yml(mfsa_id, lines) - return advisories - - def get_advisories_from_md(self, mfsa_id, lines) -> List[Advisory]: - yamltext, mdtext = split_markdown_front_matter(lines.read()) - data = yaml.safe_load(yamltext) - data["mfsa_id"] = mfsa_id - - fixed_package_urls = self.get_package_urls(data.get("fixed_in")) - references = self.get_yml_references(data) - cves = re.findall(r"CVE-\d+-\d+", yamltext + mdtext, re.IGNORECASE) - for cve in cves: - references.append( - Reference( - reference_id=cve, url=f"https://cve.mitre.org/cgi-bin/cvename.cgi?name={cve}" - ) - ) + return [] + + +def get_advisories_from_yml(mfsa_id, lines) -> List[Advisory]: + advisories = [] + data = yaml.safe_load(lines) + data["mfsa_id"] = mfsa_id - description = self.html_get_p_under_h3(markdown(mdtext), "description") + fixed_package_urls = get_package_urls(data.get("fixed_in")) + references = get_yml_references(data) - return [ + if not data.get("advisories"): + return [] + + for cve, advisory in data["advisories"].items(): + # These may contain HTML tags + summary = BeautifulSoup(advisory.get("description", ""), features="lxml").get_text() + + advisories.append( Advisory( - summary=description, - vulnerability_id="", + summary=summary, + vulnerability_id=cve if is_cve(cve) else "", impacted_package_urls=[], resolved_package_urls=fixed_package_urls, references=references, ) - ] - - def html_get_p_under_h3(self, html, h3: str): - soup = BeautifulSoup(html, features="lxml") - h3tag = soup.find("h3", text=lambda txt: txt.lower() == h3) - p = "" - if h3tag: - for tag in h3tag.next_siblings: - if tag.name: - if tag.name != "p": - break - p += tag.get_text() - return p - - def mfsa_id_from_filename(self, filename): - match = MFSA_FILENAME_RE.search(filename) - if match: - return "mfsa" + match.group(1) - - return None - - def get_package_urls(self, pkgs: List[str]) -> List[PackageURL]: - package_urls = [ - PackageURL( - type="mozilla", - # pkg is of the form "Firefox ESR 1.21" or "Thunderbird 2.21" - name=pkg.rsplit(None, 1)[0], - version=pkg.rsplit(None, 1)[1], - ) - for pkg in pkgs - ] - return package_urls - - def get_yml_references(self, data: any) -> List[Reference]: - """ - Returns a list of references - Currently only considers the given mfsa as a reference - """ - # FIXME: Needs improvement - # Should we add 'bugs' section in references too? - # Should we add 'impact'/severity of CVE in references too? - # If yes, then fix alpine_linux importer as well - # Otherwise, do we need severity field for adversary as well? - - # FIXME: Write a helper for cvssv3.1_qr severity detection ? - severities = ["critical", "low", "high", "medium", "none"] - severity = [{severity in data.get("impact"): severity} for severity in severities][0].get( - True ) - return [ + return advisories + + +def get_advisories_from_md(mfsa_id, lines) -> List[Advisory]: + yamltext, mdtext = split_markdown_front_matter(lines.read()) + data = yaml.safe_load(yamltext) + data["mfsa_id"] = mfsa_id + + fixed_package_urls = get_package_urls(data.get("fixed_in")) + references = get_yml_references(data) + cves = re.findall(r"CVE-\d+-\d+", yamltext + mdtext, re.IGNORECASE) + for cve in cves: + references.append( Reference( - reference_id=data["mfsa_id"], - url="https://www.mozilla.org/en-US/security/advisories/{}".format(data["mfsa_id"]), - severities=[VulnerabilitySeverity(scoring_systems["cvssv3.1_qr"], severity)], + reference_id=cve, + url=f"https://cve.mitre.org/cgi-bin/cvename.cgi?name={cve}", ) - ] + ) + + description = html_get_p_under_h3(markdown(mdtext), "description") + + return [ + Advisory( + summary=description, + vulnerability_id="", + impacted_package_urls=[], + resolved_package_urls=fixed_package_urls, + references=references, + ) + ] + + +def html_get_p_under_h3(html, h3: str): + soup = BeautifulSoup(html, features="lxml") + h3tag = soup.find("h3", text=lambda txt: txt.lower() == h3) + p = "" + if h3tag: + for tag in h3tag.next_siblings: + if tag.name: + if tag.name != "p": + break + p += tag.get_text() + return p + + +def mfsa_id_from_filename(filename): + match = MFSA_FILENAME_RE.search(filename) + if match: + return "mfsa" + match.group(1) + + return None + + +def get_package_urls(pkgs: List[str]) -> List[PackageURL]: + package_urls = [ + PackageURL( + type="mozilla", + # pkg is of the form "Firefox ESR 1.21" or "Thunderbird 2.21" + name=pkg.rsplit(None, 1)[0], + version=pkg.rsplit(None, 1)[1], + ) + for pkg in pkgs + if pkg + ] + return package_urls + + +def get_yml_references(data: any) -> List[Reference]: + """ + Returns a list of references + Currently only considers the given mfsa as a reference + """ + # FIXME: Needs improvement + # Should we add 'bugs' section in references too? + # Should we add 'impact'/severity of CVE in references too? + # If yes, then fix alpine_linux importer as well + # Otherwise, do we need severity field for adversary as well? + + severities = ["critical", "high", "medium", "low", "none"] + severity = "none" + if data.get("impact"): + impact = data.get("impact").lower() + for s in severities: + if s in impact: + severity = s + break + + return [ + Reference( + reference_id=data["mfsa_id"], + url="https://www.mozilla.org/en-US/security/advisories/{}".format(data["mfsa_id"]), + # severities=[VulnerabilitySeverity(scoring_systems["unspecified"], severity)], + severities=[VulnerabilitySeverity(scoring_systems["cvssv3.1_qr"], severity)], + ) + ] From c60ab41a44d80cddf37716638180ba3f5f8833c8 Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Wed, 31 Mar 2021 00:40:16 +0530 Subject: [PATCH 12/16] Remove dangling pdb comment Signed-off-by: Hritik Vijay --- vulnerabilities/tests/test_mozilla.py | 1 - 1 file changed, 1 deletion(-) diff --git a/vulnerabilities/tests/test_mozilla.py b/vulnerabilities/tests/test_mozilla.py index 5e20f83c3..d5839f792 100644 --- a/vulnerabilities/tests/test_mozilla.py +++ b/vulnerabilities/tests/test_mozilla.py @@ -53,7 +53,6 @@ def test_import(self, _): # with patch("vulnerabilities.importers.MozillaDataSource.set_api"): # runner.run() runner.run() - # pdb.set_trace() assert models.Vulnerability.objects.count() == 9 assert models.VulnerabilityReference.objects.count() == 10 From 58bcdb28ff323232906b893718aa9c468304598d Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Fri, 2 Apr 2021 14:59:03 +0530 Subject: [PATCH 13/16] reorder python imports I would really write a test case for this someday too Signed-off-by: Hritik Vijay --- vulnerabilities/tests/test_mozilla.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/tests/test_mozilla.py b/vulnerabilities/tests/test_mozilla.py index d5839f792..5d57fb0a5 100644 --- a/vulnerabilities/tests/test_mozilla.py +++ b/vulnerabilities/tests/test_mozilla.py @@ -1,8 +1,8 @@ -import shutil import os +import shutil import tempfile -from unittest.mock import patch import zipfile +from unittest.mock import patch from django.test import TestCase From 5997b26ba35d3cff83c07ee6f6fd2c81b4f526ae Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Fri, 2 Apr 2021 15:05:41 +0530 Subject: [PATCH 14/16] Use generic_textual scoring system Signed-off-by: Hritik Vijay --- vulnerabilities/importers/mozilla.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vulnerabilities/importers/mozilla.py b/vulnerabilities/importers/mozilla.py index 4b82a14bb..52535693b 100644 --- a/vulnerabilities/importers/mozilla.py +++ b/vulnerabilities/importers/mozilla.py @@ -175,7 +175,6 @@ def get_yml_references(data: any) -> List[Reference]: Reference( reference_id=data["mfsa_id"], url="https://www.mozilla.org/en-US/security/advisories/{}".format(data["mfsa_id"]), - # severities=[VulnerabilitySeverity(scoring_systems["unspecified"], severity)], - severities=[VulnerabilitySeverity(scoring_systems["cvssv3.1_qr"], severity)], + severities=[VulnerabilitySeverity(scoring_systems["generic_textual"], severity)], ) ] From 62a27891e2f0174fda93ea170eb2916b5211d506 Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Tue, 8 Feb 2022 15:46:18 +0530 Subject: [PATCH 15/16] Blackify and ignore mozilla tests Signed-off-by: Hritik Vijay --- pytest.ini | 1 + vulnerabilities/helpers.py | 2 +- vulnerabilities/tests/conftest.py | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 7a196de40..dea1a69ea 100644 --- a/pytest.ini +++ b/pytest.ini @@ -30,5 +30,6 @@ addopts = --ignore=vulnerabilities/importers/suse_backports.py --ignore=vulnerabilities/importers/suse_scores.py --ignore=vulnerabilities/importers/ubuntu_usn.py + --ignore=vulnerabilities/importers/mozilla.py --ignore=vulnerabilities/management/commands/create_cpe_to_purl_map.py --ignore=vulnerabilities/lib_oval.py diff --git a/vulnerabilities/helpers.py b/vulnerabilities/helpers.py index a5a4a622e..b43cc0242 100644 --- a/vulnerabilities/helpers.py +++ b/vulnerabilities/helpers.py @@ -72,7 +72,6 @@ def fetch_yaml(url): create_etag = MagicMock() - def split_markdown_front_matter(lines: str) -> Tuple[str, str]: """ This function splits lines into markdown front matter and the markdown body @@ -107,6 +106,7 @@ def split_markdown_front_matter(lines: str) -> Tuple[str, str]: return "\n".join(fmlines), "\n".join(mdlines) + def contains_alpha(string): """ Return True if the input 'string' contains any alphabet diff --git a/vulnerabilities/tests/conftest.py b/vulnerabilities/tests/conftest.py index 033b90539..d6072ff86 100644 --- a/vulnerabilities/tests/conftest.py +++ b/vulnerabilities/tests/conftest.py @@ -74,4 +74,5 @@ def no_rmtree(monkeypatch): "test_importer_yielder.py", "test_upstream.py", "test_istio.py", + "test_mozilla.py", ] From e6d652c068e56d037bbb72ce183aa5e6454eda38 Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Tue, 8 Feb 2022 23:01:24 +0530 Subject: [PATCH 16/16] Rename DataSource -> Importer Signed-off-by: Hritik Vijay --- vulnerabilities/importers/mozilla.py | 14 +++++++------- vulnerabilities/tests/test_mozilla.py | 10 +++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/vulnerabilities/importers/mozilla.py b/vulnerabilities/importers/mozilla.py index 52535693b..d3bbbd0c8 100644 --- a/vulnerabilities/importers/mozilla.py +++ b/vulnerabilities/importers/mozilla.py @@ -7,21 +7,21 @@ from markdown import markdown from packageurl import PackageURL -from vulnerabilities.data_source import Advisory -from vulnerabilities.data_source import GitDataSource -from vulnerabilities.data_source import Reference -from vulnerabilities.data_source import VulnerabilitySeverity +from vulnerabilities.importer import Advisory +from vulnerabilities.importer import GitImporter +from vulnerabilities.importer import Reference +from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.helpers import is_cve from vulnerabilities.helpers import split_markdown_front_matter -from vulnerabilities.severity_systems import scoring_systems +from vulnerabilities.severity_systems import SCORING_SYSTEMS REPOSITORY = "mozilla/foundation-security-advisories" MFSA_FILENAME_RE = re.compile(r"mfsa(\d{4}-\d{2,3})\.(md|yml)$") -class MozillaDataSource(GitDataSource): +class MozillaImporter(GitImporter): def __enter__(self): - super(MozillaDataSource, self).__enter__() + super(MozillaImporter, self).__enter__() if not getattr(self, "_added_files", None): self._added_files, self._updated_files = self.file_changes( diff --git a/vulnerabilities/tests/test_mozilla.py b/vulnerabilities/tests/test_mozilla.py index 5d57fb0a5..83e7435bc 100644 --- a/vulnerabilities/tests/test_mozilla.py +++ b/vulnerabilities/tests/test_mozilla.py @@ -14,7 +14,7 @@ TEST_DATA = os.path.join(BASE_DIR, "test_data/") -@patch("vulnerabilities.importers.MozillaDataSource._update_from_remote") +@patch("vulnerabilities.importers.MozillaImporter._update_from_remote") class MozillaImportTest(TestCase): tempdir = None @@ -31,7 +31,7 @@ def setUpClass(cls) -> None: name="mozilla_unittests", license="", last_run=None, - data_source="MozillaDataSource", + data_source="MozillaImporter", data_source_cfg={ "repository_url": "https://example.git", "working_directory": os.path.join(cls.tempdir, "mozilla_test"), @@ -48,9 +48,9 @@ def tearDownClass(cls) -> None: def test_import(self, _): runner = ImportRunner(self.importer, 100) - # Remove if we don't need set_api in MozillaDataSource - # with patch("vulnerabilities.importers.MozillaDataSource.versions", new=MOCK_VERSION_API): - # with patch("vulnerabilities.importers.MozillaDataSource.set_api"): + # Remove if we don't need set_api in MozillaImporter + # with patch("vulnerabilities.importers.MozillaImporter.versions", new=MOCK_VERSION_API): + # with patch("vulnerabilities.importers.MozillaImporter.set_api"): # runner.run() runner.run()