From 97cecbadda4beb9a2db63f0a76209377580c7515 Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Mon, 22 Mar 2021 02:41:30 +0530 Subject: [PATCH 1/7] Implement added_advisories() method As all the files in istio git repo are added and not updated, it is mandatory to handle `self._added_files` properly which was ignored earlier. This fixes #394 Signed-off-by: Hritik Vijay --- vulnerabilities/importers/istio.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/vulnerabilities/importers/istio.py b/vulnerabilities/importers/istio.py index fa0f1f903..b44d20a38 100644 --- a/vulnerabilities/importers/istio.py +++ b/vulnerabilities/importers/istio.py @@ -31,6 +31,7 @@ from vulnerabilities.data_source import Advisory, GitDataSource, Reference from vulnerabilities.package_managers import GitHubTagsAPI +IS_RELEASE = re.compile(r"^[\d.]+$", re.IGNORECASE).match class IstioDataSource(GitDataSource): def __enter__(self): @@ -46,8 +47,13 @@ def __enter__(self): def set_api(self): asyncio.run(self.version_api.load_api(["istio/istio"])) + def added_advisories(self) -> Set[Advisory]: + return self._load_advisories(self._added_files) + def updated_advisories(self) -> Set[Advisory]: - files = self._updated_files + return self._load_advisories(self._updated_files) + + def _load_advisories(self, files) -> Set[Advisory]: advisories = [] for f in files: processed_data = self.process_file(f) @@ -131,7 +137,7 @@ def process_file(self, path): ubound = "<=" + release[2] releases.append(lbound + "," + ubound) # If it is a single release - elif is_release(release): + elif IS_RELEASE(release): releases.append(release) data["release_ranges"] = releases @@ -195,5 +201,3 @@ def get_data_from_md(self, path): with open(path) as f: yaml_lines = self.get_yaml_lines(f) return self.get_data_from_yaml_lines(yaml_lines) - - is_release = re.compile(r"^[\d.]+$", re.IGNORECASE).match From bc3b399a4f9292bb58885bceed7e0e16c5f3dbd0 Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Mon, 22 Mar 2021 02:46:11 +0530 Subject: [PATCH 2/7] Fix package url call type was used where name had been and name where type should be. Signed-off-by: Hritik Vijay --- vulnerabilities/importers/istio.py | 10 ++-- vulnerabilities/tests/test_istio.py | 72 ++++++++++++++--------------- 2 files changed, 42 insertions(+), 40 deletions(-) diff --git a/vulnerabilities/importers/istio.py b/vulnerabilities/importers/istio.py index b44d20a38..4aa52d535 100644 --- a/vulnerabilities/importers/istio.py +++ b/vulnerabilities/importers/istio.py @@ -48,6 +48,8 @@ def set_api(self): asyncio.run(self.version_api.load_api(["istio/istio"])) def added_advisories(self) -> Set[Advisory]: + import pdb + #pdb.set_trace() return self._load_advisories(self._added_files) def updated_advisories(self) -> Set[Advisory]: @@ -161,23 +163,23 @@ def process_file(self, path): ) safe_purls_golang = { - PackageURL(type="golang", name="istio", version=version) + PackageURL(name="golang", type="istio", version=version) for version in safe_pkg_versions } safe_purls_github = { - PackageURL(type="github", name="istio", version=version) + PackageURL(name="github", type="istio", version=version) for version in safe_pkg_versions } safe_purls = safe_purls_github.union(safe_purls_golang) vuln_purls_golang = { - PackageURL(type="golang", name="istio", version=version) + PackageURL(name="golang", type="istio", version=version) for version in vuln_pkg_versions } vuln_purls_github = { - PackageURL(type="github", name="istio", version=version) + PackageURL(name="github", type="istio", version=version) for version in vuln_pkg_versions } vuln_purls = vuln_purls_github.union(vuln_purls_golang) diff --git a/vulnerabilities/tests/test_istio.py b/vulnerabilities/tests/test_istio.py index b907afe0f..38a49e632 100644 --- a/vulnerabilities/tests/test_istio.py +++ b/vulnerabilities/tests/test_istio.py @@ -81,95 +81,95 @@ def test_process_file(self): summary=("Incorrect access control."), impacted_package_urls={ PackageURL( - type="golang", - name="istio", + name="golang", + type="istio", version="1.1.0-snapshot.2", ), PackageURL( - type="golang", - name="istio", + name="golang", + type="istio", version="1.1.0-snapshot.3", ), PackageURL( - type="github", - name="istio", + name="github", + type="istio", version="1.1.0-snapshot.2", ), PackageURL( - type="github", - name="istio", + name="github", + type="istio", version="1.1.0-snapshot.3", ), }, resolved_package_urls={ PackageURL( - type="golang", - name="istio", + name="golang", + type="istio", version="1.1.0-rc.2", ), PackageURL( - type="golang", - name="istio", + name="golang", + type="istio", version="1.1.0-rc.4", ), PackageURL( - type="golang", - name="istio", + name="golang", + type="istio", version="1.1.0-rc.3", ), PackageURL( - type="golang", - name="istio", + name="golang", + type="istio", version="1.1.0-rc.0", ), PackageURL( - type="golang", - name="istio", + name="golang", + type="istio", version="1.1.0-rc.5", ), PackageURL( - type="golang", - name="istio", + name="golang", + type="istio", version="1.1.0-rc.1", ), PackageURL( - type="golang", - name="istio", + name="golang", + type="istio", version="1.1.0-rc.6", ), PackageURL( - type="github", - name="istio", + name="github", + type="istio", version="1.1.0-rc.2", ), PackageURL( - type="github", - name="istio", + name="github", + type="istio", version="1.1.0-rc.4", ), PackageURL( - type="github", - name="istio", + name="github", + type="istio", version="1.1.0-rc.3", ), PackageURL( - type="github", - name="istio", + name="github", + type="istio", version="1.1.0-rc.0", ), PackageURL( - type="github", - name="istio", + name="github", + type="istio", version="1.1.0-rc.5", ), PackageURL( - type="github", - name="istio", + name="github", + type="istio", version="1.1.0-rc.1", ), PackageURL( - type="github", - name="istio", + name="github", + type="istio", version="1.1.0-rc.6", ), }, From 08b72ef3b3cc7c9eab27ce73b47695df98b5988b Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Mon, 22 Mar 2021 02:55:00 +0530 Subject: [PATCH 3/7] Remove redundent comment Signed-off-by: Hritik Vijay --- vulnerabilities/importers/istio.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vulnerabilities/importers/istio.py b/vulnerabilities/importers/istio.py index 4aa52d535..0ee116fea 100644 --- a/vulnerabilities/importers/istio.py +++ b/vulnerabilities/importers/istio.py @@ -33,6 +33,7 @@ IS_RELEASE = re.compile(r"^[\d.]+$", re.IGNORECASE).match + class IstioDataSource(GitDataSource): def __enter__(self): super(IstioDataSource, self).__enter__() @@ -48,8 +49,6 @@ def set_api(self): asyncio.run(self.version_api.load_api(["istio/istio"])) def added_advisories(self) -> Set[Advisory]: - import pdb - #pdb.set_trace() return self._load_advisories(self._added_files) def updated_advisories(self) -> Set[Advisory]: From 6b24cd81545df98b2d5afe2a4b60a371e3004144 Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Mon, 22 Mar 2021 02:46:11 +0530 Subject: [PATCH 4/7] Fix package url call type was used where name had been and name where type should be. Signed-off-by: Hritik Vijay --- vulnerabilities/importers/istio.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vulnerabilities/importers/istio.py b/vulnerabilities/importers/istio.py index 0ee116fea..6aac7eb32 100644 --- a/vulnerabilities/importers/istio.py +++ b/vulnerabilities/importers/istio.py @@ -49,6 +49,8 @@ def set_api(self): asyncio.run(self.version_api.load_api(["istio/istio"])) def added_advisories(self) -> Set[Advisory]: + import pdb + #pdb.set_trace() return self._load_advisories(self._added_files) def updated_advisories(self) -> Set[Advisory]: From 12a2a0d97e3e43a30a7328a502ea6dcd2f99f1bc Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Mon, 22 Mar 2021 02:59:57 +0530 Subject: [PATCH 5/7] Remove unnecessary pdb import Signed-off-by: Hritik Vijay --- vulnerabilities/importers/istio.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/vulnerabilities/importers/istio.py b/vulnerabilities/importers/istio.py index 6aac7eb32..0ee116fea 100644 --- a/vulnerabilities/importers/istio.py +++ b/vulnerabilities/importers/istio.py @@ -49,8 +49,6 @@ def set_api(self): asyncio.run(self.version_api.load_api(["istio/istio"])) def added_advisories(self) -> Set[Advisory]: - import pdb - #pdb.set_trace() return self._load_advisories(self._added_files) def updated_advisories(self) -> Set[Advisory]: From f0215462b5dbd475eb58b630e62b12354dd84e4b Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Thu, 25 Mar 2021 18:11:03 +0530 Subject: [PATCH 6/7] Refactor:packageurl repr,is_release,rm added_files 1. The package url call should have order similar to it's string repr (type comes first). 2. Upper casing a function variable is just misleading. Upper case means it's some global dataish thing (is_release) 3. Remove added_files function Reference: https://github.com/nexB/vulnerablecode/pull/395#issuecomment-803957612 Signed-off-by: Hritik Vijay --- vulnerabilities/importers/istio.py | 19 ++++++--------- vulnerabilities/tests/test_istio.py | 36 ++++++++++++++--------------- 2 files changed, 25 insertions(+), 30 deletions(-) diff --git a/vulnerabilities/importers/istio.py b/vulnerabilities/importers/istio.py index 0ee116fea..eeb39dc58 100644 --- a/vulnerabilities/importers/istio.py +++ b/vulnerabilities/importers/istio.py @@ -31,7 +31,7 @@ from vulnerabilities.data_source import Advisory, GitDataSource, Reference from vulnerabilities.package_managers import GitHubTagsAPI -IS_RELEASE = re.compile(r"^[\d.]+$", re.IGNORECASE).match +is_release = re.compile(r"^[\d.]+$", re.IGNORECASE).match class IstioDataSource(GitDataSource): @@ -48,13 +48,8 @@ def __enter__(self): def set_api(self): asyncio.run(self.version_api.load_api(["istio/istio"])) - def added_advisories(self) -> Set[Advisory]: - return self._load_advisories(self._added_files) - def updated_advisories(self) -> Set[Advisory]: - return self._load_advisories(self._updated_files) - - def _load_advisories(self, files) -> Set[Advisory]: + files = self._added_files.union(self._updated_files) advisories = [] for f in files: processed_data = self.process_file(f) @@ -138,7 +133,7 @@ def process_file(self, path): ubound = "<=" + release[2] releases.append(lbound + "," + ubound) # If it is a single release - elif IS_RELEASE(release): + elif is_release(release): releases.append(release) data["release_ranges"] = releases @@ -162,23 +157,23 @@ def process_file(self, path): ) safe_purls_golang = { - PackageURL(name="golang", type="istio", version=version) + PackageURL(type="istio", name="golang", version=version) for version in safe_pkg_versions } safe_purls_github = { - PackageURL(name="github", type="istio", version=version) + PackageURL(type="istio", name="github", version=version) for version in safe_pkg_versions } safe_purls = safe_purls_github.union(safe_purls_golang) vuln_purls_golang = { - PackageURL(name="golang", type="istio", version=version) + PackageURL(type="istio", name="golang", version=version) for version in vuln_pkg_versions } vuln_purls_github = { - PackageURL(name="github", type="istio", version=version) + PackageURL(type="istio", name="github", version=version) for version in vuln_pkg_versions } vuln_purls = vuln_purls_github.union(vuln_purls_golang) diff --git a/vulnerabilities/tests/test_istio.py b/vulnerabilities/tests/test_istio.py index 38a49e632..f0bddd594 100644 --- a/vulnerabilities/tests/test_istio.py +++ b/vulnerabilities/tests/test_istio.py @@ -81,95 +81,95 @@ def test_process_file(self): summary=("Incorrect access control."), impacted_package_urls={ PackageURL( - name="golang", type="istio", + name="golang", version="1.1.0-snapshot.2", ), PackageURL( - name="golang", type="istio", + name="golang", version="1.1.0-snapshot.3", ), PackageURL( - name="github", type="istio", + name="github", version="1.1.0-snapshot.2", ), PackageURL( - name="github", type="istio", + name="github", version="1.1.0-snapshot.3", ), }, resolved_package_urls={ PackageURL( - name="golang", type="istio", + name="golang", version="1.1.0-rc.2", ), PackageURL( - name="golang", type="istio", + name="golang", version="1.1.0-rc.4", ), PackageURL( - name="golang", type="istio", + name="golang", version="1.1.0-rc.3", ), PackageURL( - name="golang", type="istio", + name="golang", version="1.1.0-rc.0", ), PackageURL( - name="golang", type="istio", + name="golang", version="1.1.0-rc.5", ), PackageURL( - name="golang", type="istio", + name="golang", version="1.1.0-rc.1", ), PackageURL( - name="golang", type="istio", + name="golang", version="1.1.0-rc.6", ), PackageURL( - name="github", type="istio", + name="github", version="1.1.0-rc.2", ), PackageURL( - name="github", type="istio", + name="github", version="1.1.0-rc.4", ), PackageURL( - name="github", type="istio", + name="github", version="1.1.0-rc.3", ), PackageURL( - name="github", type="istio", + name="github", version="1.1.0-rc.0", ), PackageURL( - name="github", type="istio", + name="github", version="1.1.0-rc.5", ), PackageURL( - name="github", type="istio", + name="github", version="1.1.0-rc.1", ), PackageURL( - name="github", type="istio", + name="github", version="1.1.0-rc.6", ), }, From 706986eb2e698ddb5ef5bb92ba71feee998ecf8a Mon Sep 17 00:00:00 2001 From: Hritik Vijay Date: Sat, 27 Mar 2021 02:33:57 +0530 Subject: [PATCH 7/7] pkg name="istio" type="golang/github/etc" Signed-off-by: Hritik Vijay --- vulnerabilities/importers/istio.py | 8 ++-- vulnerabilities/tests/test_istio.py | 72 ++++++++++++++--------------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/vulnerabilities/importers/istio.py b/vulnerabilities/importers/istio.py index eeb39dc58..5e29e739b 100644 --- a/vulnerabilities/importers/istio.py +++ b/vulnerabilities/importers/istio.py @@ -157,23 +157,23 @@ def process_file(self, path): ) safe_purls_golang = { - PackageURL(type="istio", name="golang", version=version) + PackageURL(type="golang", name="istio", version=version) for version in safe_pkg_versions } safe_purls_github = { - PackageURL(type="istio", name="github", version=version) + PackageURL(type="github", name="istio", version=version) for version in safe_pkg_versions } safe_purls = safe_purls_github.union(safe_purls_golang) vuln_purls_golang = { - PackageURL(type="istio", name="golang", version=version) + PackageURL(type="golang", name="istio", version=version) for version in vuln_pkg_versions } vuln_purls_github = { - PackageURL(type="istio", name="github", version=version) + PackageURL(type="github", name="istio", version=version) for version in vuln_pkg_versions } vuln_purls = vuln_purls_github.union(vuln_purls_golang) diff --git a/vulnerabilities/tests/test_istio.py b/vulnerabilities/tests/test_istio.py index f0bddd594..b907afe0f 100644 --- a/vulnerabilities/tests/test_istio.py +++ b/vulnerabilities/tests/test_istio.py @@ -81,95 +81,95 @@ def test_process_file(self): summary=("Incorrect access control."), impacted_package_urls={ PackageURL( - type="istio", - name="golang", + type="golang", + name="istio", version="1.1.0-snapshot.2", ), PackageURL( - type="istio", - name="golang", + type="golang", + name="istio", version="1.1.0-snapshot.3", ), PackageURL( - type="istio", - name="github", + type="github", + name="istio", version="1.1.0-snapshot.2", ), PackageURL( - type="istio", - name="github", + type="github", + name="istio", version="1.1.0-snapshot.3", ), }, resolved_package_urls={ PackageURL( - type="istio", - name="golang", + type="golang", + name="istio", version="1.1.0-rc.2", ), PackageURL( - type="istio", - name="golang", + type="golang", + name="istio", version="1.1.0-rc.4", ), PackageURL( - type="istio", - name="golang", + type="golang", + name="istio", version="1.1.0-rc.3", ), PackageURL( - type="istio", - name="golang", + type="golang", + name="istio", version="1.1.0-rc.0", ), PackageURL( - type="istio", - name="golang", + type="golang", + name="istio", version="1.1.0-rc.5", ), PackageURL( - type="istio", - name="golang", + type="golang", + name="istio", version="1.1.0-rc.1", ), PackageURL( - type="istio", - name="golang", + type="golang", + name="istio", version="1.1.0-rc.6", ), PackageURL( - type="istio", - name="github", + type="github", + name="istio", version="1.1.0-rc.2", ), PackageURL( - type="istio", - name="github", + type="github", + name="istio", version="1.1.0-rc.4", ), PackageURL( - type="istio", - name="github", + type="github", + name="istio", version="1.1.0-rc.3", ), PackageURL( - type="istio", - name="github", + type="github", + name="istio", version="1.1.0-rc.0", ), PackageURL( - type="istio", - name="github", + type="github", + name="istio", version="1.1.0-rc.5", ), PackageURL( - type="istio", - name="github", + type="github", + name="istio", version="1.1.0-rc.1", ), PackageURL( - type="istio", - name="github", + type="github", + name="istio", version="1.1.0-rc.6", ), },