From 9f5aaa66b3c8bb7307bef349a766ec248cafb2a3 Mon Sep 17 00:00:00 2001 From: jboursier Date: Thu, 20 Oct 2022 18:48:51 +0200 Subject: [PATCH 01/13] Initial support for Dependabot commands Signed-off-by: jboursier --- src/cli.py | 35 ++++++++++++++++++++++++++++++- src/ghas_cli/utils/dependabot.py | 36 ++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 src/ghas_cli/utils/dependabot.py diff --git a/src/cli.py b/src/cli.py index a7a7ab8..e693757 100644 --- a/src/cli.py +++ b/src/cli.py @@ -18,7 +18,7 @@ print("Missing dependencies. Please reach @jboursier if needed.") sys.exit(255) -from ghas_cli.utils import repositories, vulns, teams, issues, actions +from ghas_cli.utils import repositories, vulns, teams, issues, actions, dependabot def main() -> None: @@ -583,6 +583,39 @@ def dependabot_alerts() -> None: pass +@dependabot_alerts.command("list") +@click.option( + "-r", + "--repository", + prompt="Repository name", +) +@click.option( + "-t", + "--token", + prompt=False, + type=str, + default=None, + hide_input=True, + confirmation_prompt=False, + show_envvar=True, +) +@click.option("-o", "--organization", prompt="Organization name", type=str) +def dependabot_alerts_list( + repository: str, + organization: str, + token: str, +) -> None: + """Get issues created by an user on a repository""" + + dependabot_res = dependabot.list_alerts_repo( + repository=repository, + organization=organization, + token=token, + ) + + click.echo(dependabot_res) + + ########### # Actions # ########### diff --git a/src/ghas_cli/utils/dependabot.py b/src/ghas_cli/utils/dependabot.py new file mode 100644 index 0000000..dcd04ac --- /dev/null +++ b/src/ghas_cli/utils/dependabot.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python3 + +from typing import Dict, List +import requests + +from . import network + + +def list_alerts_repo(repository: str, organization: str, token: str) -> Dict: + """Get Dependabot alerts for one repository""" + + headers = network.get_github_headers(token) + + alerts_repo = [] + page = 1 + while True: + params = {"state": "open", "per_page": 100, "page": page} + alerts = requests.get( + url=f"https://api.github.com/repos/{organization}/{repository}/dependabot/alerts", + params=params, + headers=headers, + ) + if network.check_rate_limit(alerts): + break + if alerts.status_code != 200: + break + if not alerts.json(): + break + for a in alerts.json(): + if not a: + continue + alerts_repo.append(alerts.json()) + page += 1 + + return alerts_repo From 0a7af42a4a84d54e35360007798351f5782c87b0 Mon Sep 17 00:00:00 2001 From: jboursier Date: Thu, 20 Oct 2022 22:01:23 +0200 Subject: [PATCH 02/13] Fixes Signed-off-by: jboursier --- src/cli.py | 3 ++- src/ghas_cli/utils/dependabot.py | 28 ++++++++++++++++++---------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/cli.py b/src/cli.py index e693757..1835f4c 100644 --- a/src/cli.py +++ b/src/cli.py @@ -613,7 +613,8 @@ def dependabot_alerts_list( token=token, ) - click.echo(dependabot_res) + for res in dependabot_res: + click.echo(res) ########### diff --git a/src/ghas_cli/utils/dependabot.py b/src/ghas_cli/utils/dependabot.py index dcd04ac..d21b315 100644 --- a/src/ghas_cli/utils/dependabot.py +++ b/src/ghas_cli/utils/dependabot.py @@ -3,11 +3,13 @@ from typing import Dict, List import requests +import json +import time from . import network -def list_alerts_repo(repository: str, organization: str, token: str) -> Dict: +def list_alerts_repo(repository: str, organization: str, token: str) -> List: """Get Dependabot alerts for one repository""" headers = network.get_github_headers(token) @@ -15,14 +17,20 @@ def list_alerts_repo(repository: str, organization: str, token: str) -> Dict: alerts_repo = [] page = 1 while True: - params = {"state": "open", "per_page": 100, "page": page} - alerts = requests.get( - url=f"https://api.github.com/repos/{organization}/{repository}/dependabot/alerts", - params=params, - headers=headers, - ) - if network.check_rate_limit(alerts): - break + i = 0 + while i < network.RETRIES: + params = {"state": "open", "per_page": 100, "page": page} + alerts = requests.get( + url=f"https://api.github.com/repos/{organization}/{repository}/dependabot/alerts", + params=params, + headers=headers, + ) + if network.check_rate_limit(alerts): + time.sleep(network.SLEEP_1_MINUTE) + i += 1 + else: + break + if alerts.status_code != 200: break if not alerts.json(): @@ -30,7 +38,7 @@ def list_alerts_repo(repository: str, organization: str, token: str) -> Dict: for a in alerts.json(): if not a: continue - alerts_repo.append(alerts.json()) + alerts_repo.append(json.dumps(a)) page += 1 return alerts_repo From 6a5ddcf5ab4a342ee76ab8ec1903c56e94840743 Mon Sep 17 00:00:00 2001 From: jboursier Date: Fri, 21 Oct 2022 18:35:27 +0200 Subject: [PATCH 03/13] Support multiple repositories to be passed on the CLI Signed-off-by: jboursier --- src/cli.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/cli.py b/src/cli.py index 1835f4c..f886c49 100644 --- a/src/cli.py +++ b/src/cli.py @@ -10,7 +10,7 @@ try: import click - from typing import Dict, Any + from typing import Dict, Any, List from datetime import datetime except ImportError: import sys @@ -586,8 +586,10 @@ def dependabot_alerts() -> None: @dependabot_alerts.command("list") @click.option( "-r", - "--repository", - prompt="Repository name", + "--repos", + prompt="Repositories name. Use `all` to retrieve alerts for all repos.", + type=str, + multiple=True, ) @click.option( "-t", @@ -601,20 +603,21 @@ def dependabot_alerts() -> None: ) @click.option("-o", "--organization", prompt="Organization name", type=str) def dependabot_alerts_list( - repository: str, + repos: List, organization: str, token: str, ) -> None: - """Get issues created by an user on a repository""" + """Get Dependabot alerts for a repository""" - dependabot_res = dependabot.list_alerts_repo( - repository=repository, - organization=organization, - token=token, - ) + for repo in repos: + dependabot_res = dependabot.list_alerts_repo( + repository=repository, + organization=organization, + token=token, + ) - for res in dependabot_res: - click.echo(res) + for res in dependabot_res: + click.echo(res) ########### From 4007fa40f83c1ca2bc6aabe90ee7e2af8bc56e0a Mon Sep 17 00:00:00 2001 From: jboursier Date: Wed, 24 May 2023 15:49:18 +0200 Subject: [PATCH 04/13] Support exporting SBOM in various formats Signed-off-by: jboursier --- src/cli.py | 33 +++++++++++++++++++++++++ src/ghas_cli/utils/dependabot.py | 42 ++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/src/cli.py b/src/cli.py index 343b7f7..efc3f49 100644 --- a/src/cli.py +++ b/src/cli.py @@ -847,6 +847,39 @@ def dependabot_alerts_list( click.echo(res) +@dependabot_alerts.command("get_dependencies") +@click.option( + "-f", + "--format", + prompt="Output format", + type=click.Choice( + ["sbom", "csv", "txt"], + case_sensitive=True, + ), + default="sbom", +) +@click.option( + "-r", + "--repository", + prompt="Repository name", +) +@click.option( + "-t", + "--token", + prompt=False, + type=str, + default=None, + hide_input=True, + confirmation_prompt=False, + show_envvar=True, +) +@click.option("-o", "--organization", prompt="Organization name", type=str) +def dependabot_get_dependencies(repository:str, organization:str, token:str, format: str="sbom") -> None: + """Get a list of dependencies for a repository""" + + res = dependabot.get_dependencies(repository, organization, token, format=format) + click.echo(res) + ########### # Actions # ########### diff --git a/src/ghas_cli/utils/dependabot.py b/src/ghas_cli/utils/dependabot.py index d21b315..393b796 100644 --- a/src/ghas_cli/utils/dependabot.py +++ b/src/ghas_cli/utils/dependabot.py @@ -5,6 +5,7 @@ import requests import json import time +import logging from . import network @@ -42,3 +43,44 @@ def list_alerts_repo(repository: str, organization: str, token: str) -> List: page += 1 return alerts_repo + + + +def get_dependencies(repository: str, organization: str, token: str, format:str ="sbom"): + """ + Get the list of dependencies for one repository. + + Available formats: + - `sbom` - SPDX json + - `CSV` - CSV export + - `txt` - basic export + + https://docs.github.com/en/rest/dependency-graph/sboms?apiVersion=2022-11-28 + """ + headers = network.get_github_headers(token) + + dependencies = requests.get( + url=f"https://api.github.com/repos/{organization}/{repository}/dependency-graph/sbom", + headers=headers, + ) + + + if dependencies.status_code != 200: + logging.error(f"Unable to retrieve the dependencies for {repository} - {dependencies.status_code} - {dependencies.content}") + return False + + if "sbom" == format: + return dependencies.json() + elif "csv" == format: + deps = "" + for dep in dependencies.json()["sbom"]["packages"]: + deps += f"{dep['name']},{dep['versionInfo']}, {dep['licenseConcluded']}\n" + return deps + elif "txt" == format: + deps = "" + for dep in dependencies.json()["sbom"]["packages"]: + deps += dep["name"] + "\n" + return deps + else: + logging.error(f"Invalid export format {format}. Must be one of `sbom`, `csv` or `txt`.") + return False \ No newline at end of file From 8f28b14d091e6993d5032a4da13d531a7e496e6d Mon Sep 17 00:00:00 2001 From: jboursier Date: Wed, 24 May 2023 15:56:37 +0200 Subject: [PATCH 05/13] Rename Dependabot group commands Signed-off-by: jboursier --- src/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cli.py b/src/cli.py index efc3f49..46711d1 100644 --- a/src/cli.py +++ b/src/cli.py @@ -804,13 +804,13 @@ def secret_alerts_export( ############## -@cli.group() +@cli.group(name="dependabot") def dependabot_alerts() -> None: """Manage Dependabot alerts""" pass -@dependabot_alerts.command("list") +@dependabot_alerts.command("get_alerts") @click.option( "-r", "--repos", From dedbc1bd3927b24e133c7f4be69be31b86971d73 Mon Sep 17 00:00:00 2001 From: jboursier Date: Wed, 24 May 2023 16:10:02 +0200 Subject: [PATCH 06/13] Fix typo Signed-off-by: jboursier --- src/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli.py b/src/cli.py index 46711d1..1f7161e 100644 --- a/src/cli.py +++ b/src/cli.py @@ -838,7 +838,7 @@ def dependabot_alerts_list( for repo in repos: dependabot_res = dependabot.list_alerts_repo( - repository=repository, + repository=repo, organization=organization, token=token, ) From 31886847ea3fe411a7f6e99586da65ead4a4cdd1 Mon Sep 17 00:00:00 2001 From: jboursier Date: Wed, 24 May 2023 16:11:23 +0200 Subject: [PATCH 07/13] Remove unused import Signed-off-by: jboursier --- src/ghas_cli/utils/dependabot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ghas_cli/utils/dependabot.py b/src/ghas_cli/utils/dependabot.py index 393b796..be84c0f 100644 --- a/src/ghas_cli/utils/dependabot.py +++ b/src/ghas_cli/utils/dependabot.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- #!/usr/bin/env python3 -from typing import Dict, List +from typing import List import requests import json import time From 4a0e7931d384ac43131bc96c2d96c4874f796d6a Mon Sep 17 00:00:00 2001 From: jboursier Date: Mon, 18 Mar 2024 12:42:42 +0100 Subject: [PATCH 08/13] Fix the license fetching logic for sbom CSV export Signed-off-by: jboursier --- src/ghas_cli/utils/dependabot.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/ghas_cli/utils/dependabot.py b/src/ghas_cli/utils/dependabot.py index be84c0f..5359078 100644 --- a/src/ghas_cli/utils/dependabot.py +++ b/src/ghas_cli/utils/dependabot.py @@ -74,7 +74,16 @@ def get_dependencies(repository: str, organization: str, token: str, format:str elif "csv" == format: deps = "" for dep in dependencies.json()["sbom"]["packages"]: - deps += f"{dep['name']},{dep['versionInfo']}, {dep['licenseConcluded']}\n" + license = "Unknown" + try: + license = dep['licenseConcluded'] + except: + try: + license = dep['licenseDeclared'] + except: + license = "Unknown" + + deps += f"{repository}, {dep['name']},{dep['versionInfo']}, {license}\n" return deps elif "txt" == format: deps = "" From afbc75d12a7dbfa99210448ed105d7356410af9c Mon Sep 17 00:00:00 2001 From: jboursier Date: Mon, 18 Mar 2024 15:13:04 +0100 Subject: [PATCH 09/13] Support exporting topics from repositories. Support mass export of dependencies Signed-off-by: jboursier --- src/cli.py | 92 +++++++++++++++++++++++++++++- src/ghas_cli/utils/repositories.py | 20 +++++++ 2 files changed, 111 insertions(+), 1 deletion(-) diff --git a/src/cli.py b/src/cli.py index 0efbcbb..74f5520 100644 --- a/src/cli.py +++ b/src/cli.py @@ -223,6 +223,31 @@ def repositories_list( output.write(r.name + "\n") click.echo(r.name) +@repositories_cli.command("get_topics") +@click.option( + "-r", + "--repository", + prompt="Repository name", +) +@click.option( + "-t", + "--token", + prompt=False, + type=str, + default=None, + hide_input=True, + confirmation_prompt=False, + show_envvar=True, +) +@click.option("-o", "--organization", prompt="Organization name", type=str) +def repositories_get_topics( + repository: str, + organization: str, + token: str, +) -> None: + """Get a repository topics""" + click.echo(repositories.get_topics(organization, token, repository)) + @repositories_cli.command("enable_ss_protection") @click.option( @@ -904,7 +929,7 @@ def dependabot_get_dependencies(repository:str, organization:str, token:str, for """Get a list of dependencies for a repository""" res = dependabot.get_dependencies(repository, organization, token, format=format) - click.echo(res) + click.echo(res, nl=False) ########### # Actions # @@ -1476,5 +1501,70 @@ def mass_set_developer_role( return None + +@mass_cli.command("topics") +@click.argument("input_repos_list", type=click.File("r")) +@click.option( + "-t", + "--token", + prompt=False, + type=str, + default=None, + hide_input=True, + confirmation_prompt=False, + show_envvar=True, +) +@click.option("-o", "--organization", prompt="Organization name", type=str) +def mass_get_topics( + input_repos_list: Any, + organization: str, + token: str, +) -> None: + repos_list = input_repos_list.readlines() + + for repo in repos_list: + repo = repo.rstrip("\n") + + click.echo(f"{repo},", nl=False) + click.echo(repositories.get_topics(token, organization, repo)) + +@mass_cli.command("dependencies") +@click.argument("input_repos_list", type=click.File("r")) +@click.option( + "-t", + "--token", + prompt=False, + type=str, + default=None, + hide_input=True, + confirmation_prompt=False, + show_envvar=True, +) +@click.option("-o", "--organization", prompt="Organization name", type=str) +@click.option( + "-f", + "--format", + prompt="Output format", + type=click.Choice( + ["sbom", "csv", "txt"], + case_sensitive=True, + ), + default="csv", +) +def mass_get_dependencies( + input_repos_list: Any, + organization: str, + token: str, +) -> None: + repos_list = input_repos_list.readlines() + + for repo in repos_list: + repo = repo.rstrip("\n") + + click.echo(f"{repo},", nl=False) + click.echo(dependabot.get_dependencies(repo, organization, token, format=format)) + + + if __name__ == "__main__": main() diff --git a/src/ghas_cli/utils/repositories.py b/src/ghas_cli/utils/repositories.py index 6830f5f..7341a11 100644 --- a/src/ghas_cli/utils/repositories.py +++ b/src/ghas_cli/utils/repositories.py @@ -236,6 +236,26 @@ def get_default_branch_last_updated( branch_res["commit"]["commit"]["author"]["date"].split("T")[0], "%Y-%m-%d" ) +def get_topics( + token: str, organization: str, repository_name: str +) -> List: + """ + Return the repository topics + """ + headers = network.get_github_headers(token) + + topic_res = network.get( + url=f"https://api.github.com/repos/{organization}/{repository_name}/topics", + headers=headers, + ) + + if topic_res.status_code != 200: + return False + + topics_res = topic_res.json() + + return topics_res['names'] + def archive( organization: str, token: str, repository: str, archive: bool = True From 0a2fb7644ac3770b898085827a1ec73cd90f93bb Mon Sep 17 00:00:00 2001 From: jboursier Date: Mon, 18 Mar 2024 15:18:00 +0100 Subject: [PATCH 10/13] Fix passed credentials for the get_topics function Signed-off-by: jboursier --- src/cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cli.py b/src/cli.py index 74f5520..d3f1c7f 100644 --- a/src/cli.py +++ b/src/cli.py @@ -246,7 +246,7 @@ def repositories_get_topics( token: str, ) -> None: """Get a repository topics""" - click.echo(repositories.get_topics(organization, token, repository)) + click.echo(repositories.get_topics(token=token, organization=organization, repository_name=repository)) @repositories_cli.command("enable_ss_protection") @@ -1526,7 +1526,7 @@ def mass_get_topics( repo = repo.rstrip("\n") click.echo(f"{repo},", nl=False) - click.echo(repositories.get_topics(token, organization, repo)) + click.echo(repositories.get_topics(token=token, organization=organization, repository_name=repo)) @mass_cli.command("dependencies") @click.argument("input_repos_list", type=click.File("r")) @@ -1562,7 +1562,7 @@ def mass_get_dependencies( repo = repo.rstrip("\n") click.echo(f"{repo},", nl=False) - click.echo(dependabot.get_dependencies(repo, organization, token, format=format)) + click.echo(dependabot.get_dependencies(repository=repo, organization=organization, token=token, format=format)) From cd4d6a25b900aa70e27eb4f9992377fcaa27d0bf Mon Sep 17 00:00:00 2001 From: jboursier Date: Mon, 18 Mar 2024 17:29:56 +0100 Subject: [PATCH 11/13] Support mass export of dependencies for an org Signed-off-by: jboursier --- src/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cli.py b/src/cli.py index d3f1c7f..a51409e 100644 --- a/src/cli.py +++ b/src/cli.py @@ -1552,6 +1552,7 @@ def mass_get_topics( default="csv", ) def mass_get_dependencies( + format: str, input_repos_list: Any, organization: str, token: str, @@ -1562,7 +1563,7 @@ def mass_get_dependencies( repo = repo.rstrip("\n") click.echo(f"{repo},", nl=False) - click.echo(dependabot.get_dependencies(repository=repo, organization=organization, token=token, format=format)) + click.echo(dependabot.get_dependencies(repository=repo, organization=organization, token=token, format=format), nl=False) From 185dfcb658420beb8f25f26a076b4b9e3f1f4796 Mon Sep 17 00:00:00 2001 From: jboursier-mwb Date: Tue, 19 Mar 2024 18:09:20 +0100 Subject: [PATCH 12/13] Update dependabot.py Signed-off-by: jboursier-mwb --- src/ghas_cli/utils/dependabot.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ghas_cli/utils/dependabot.py b/src/ghas_cli/utils/dependabot.py index 5359078..939e53b 100644 --- a/src/ghas_cli/utils/dependabot.py +++ b/src/ghas_cli/utils/dependabot.py @@ -74,7 +74,6 @@ def get_dependencies(repository: str, organization: str, token: str, format:str elif "csv" == format: deps = "" for dep in dependencies.json()["sbom"]["packages"]: - license = "Unknown" try: license = dep['licenseConcluded'] except: @@ -92,4 +91,4 @@ def get_dependencies(repository: str, organization: str, token: str, format:str return deps else: logging.error(f"Invalid export format {format}. Must be one of `sbom`, `csv` or `txt`.") - return False \ No newline at end of file + return False From d08e43915d41acffe85f25c797435cc682acba29 Mon Sep 17 00:00:00 2001 From: jboursier-mwb Date: Tue, 19 Mar 2024 18:19:17 +0100 Subject: [PATCH 13/13] Do not sleep twice when hitting the rate limit Signed-off-by: jboursier-mwb --- src/ghas_cli/utils/dependabot.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ghas_cli/utils/dependabot.py b/src/ghas_cli/utils/dependabot.py index 939e53b..ad9d80c 100644 --- a/src/ghas_cli/utils/dependabot.py +++ b/src/ghas_cli/utils/dependabot.py @@ -27,7 +27,6 @@ def list_alerts_repo(repository: str, organization: str, token: str) -> List: headers=headers, ) if network.check_rate_limit(alerts): - time.sleep(network.SLEEP_1_MINUTE) i += 1 else: break