diff --git a/webhook_server_container/libs/github_api.py b/webhook_server_container/libs/github_api.py index 917e9cca..a84600a1 100644 --- a/webhook_server_container/libs/github_api.py +++ b/webhook_server_container/libs/github_api.py @@ -7,7 +7,7 @@ import time from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Tuple from fastapi import FastAPI from jira import JIRA @@ -58,7 +58,9 @@ ) from pyhelper_utils.general import ignore_exceptions from webhook_server_container.utils.dockerhub_rate_limit import DockerHub -from webhook_server_container.utils.github_repository_settings import get_repository_github_app_api +from webhook_server_container.utils.github_repository_settings import ( + get_repository_github_app_api, +) from webhook_server_container.utils.helpers import ( get_api_with_highest_rate_limit, extract_key_from_dict, @@ -115,7 +117,7 @@ def __init__(self, hook_data): self._set_log_prefix_color() # self.log_repository_features() - self.github_app_api = get_repository_github_app_api(config=self.config, repository=self.repository_full_name) + self.github_app_api = get_repository_github_app_api(config_=self.config, repository=self.repository_full_name) if not self.github_app_api: LOGGER.error( f"Repository {self.repository_full_name} not found by manage-repositories-app, " @@ -316,11 +318,17 @@ def _repo_data_from_config(self): self.repository_full_name = repo_data["name"] self.pypi = get_value_from_dicts(primary_dict=repo_data, secondary_dict=config_data, key="pypi") self.verified_job = get_value_from_dicts( - primary_dict=repo_data, secondary_dict=config_data, key="verified-job", return_on_none=True + primary_dict=repo_data, + secondary_dict=config_data, + key="verified-job", + return_on_none=True, ) self.tox_enabled = get_value_from_dicts(primary_dict=repo_data, secondary_dict=config_data, key="tox") self.tox_python_version = get_value_from_dicts( - primary_dict=repo_data, secondary_dict=config_data, key="tox-python-version", return_on_none="python" + primary_dict=repo_data, + secondary_dict=config_data, + key="tox-python-version", + return_on_none="python", ) self.slack_webhook_url = get_value_from_dicts( primary_dict=repo_data, secondary_dict=config_data, key="slack_webhook_url" @@ -364,10 +372,16 @@ def _repo_data_from_config(self): self.container_release = self.build_and_push_container.get("release") self.auto_verified_and_merged_users = get_value_from_dicts( - primary_dict=repo_data, secondary_dict=config_data, key="auto-verified-and-merged-users", return_on_none=[] + primary_dict=repo_data, + secondary_dict=config_data, + key="auto-verified-and-merged-users", + return_on_none=[], ) self.can_be_merged_required_labels = get_value_from_dicts( - primary_dict=repo_data, secondary_dict=config_data, key="can-be-merged-required-labels", return_on_none=[] + primary_dict=repo_data, + secondary_dict=config_data, + key="can-be-merged-required-labels", + return_on_none=[], ) def _get_pull_request(self, number=None): @@ -907,7 +921,8 @@ def process_pull_request_webhook_data(self): if _story_key := self.get_story_key_with_jira_connection(): LOGGER.info(f"{self.log_prefix} Closing Jira story") self.jira_conn.close_issue( - key=_story_key, comment=f"PR: {self.pull_request.title} is closed. Megred: {is_merged}" + key=_story_key, + comment=f"PR: {self.pull_request.title} is closed. Megred: {is_merged}", ) if is_merged: @@ -1070,7 +1085,7 @@ def _run_tox(self): output = { "title": "Tox", "summary": "", - "text": self.get_checkrun_text(err=err, out=out), + "text": self.get_check_run_text(err=err, out=out), } if rc: return self.set_run_tox_check_success(output=output) @@ -1092,7 +1107,7 @@ def _run_pre_commit(self): output = { "title": "Pre-Commit", "summary": "", - "text": self.get_checkrun_text(err=err, out=out), + "text": self.get_check_run_text(err=err, out=out), } if rc: return self.set_run_pre_commit_check_success(output=output) @@ -1284,7 +1299,7 @@ def cherry_pick(self, target_branch, reviewed_user=None): output = { "title": "Cherry-pick details", "summary": "", - "text": self.get_checkrun_text(err=err, out=out), + "text": self.get_check_run_text(err=err, out=out), } if rc: self.set_cherry_pick_success(output=output) @@ -1324,11 +1339,11 @@ def label_by_pull_requests_merge_state_after_merged(self): for pull_request in self.repository.get_pulls(state="open"): self.pull_request = pull_request LOGGER.info(f"{self.log_prefix} check label pull request after merge") - self.label_pull_request_by_merge_state() + self.label_pull_request_by_merge_state(_sleep=time_sleep) - def label_pull_request_by_merge_state(self, _sleep=30): + def label_pull_request_by_merge_state(self, _sleep=0): if _sleep: - LOGGER.info(f"{self.log_prefix} Sleep for 30 seconds before checking merge state") + LOGGER.info(f"{self.log_prefix} Sleep for {_sleep} seconds before checking merge state") time.sleep(_sleep) merge_state = self.pull_request.mergeable_state @@ -1535,7 +1550,7 @@ def _run_build_container( output = { "title": "Build container", "summary": "", - "text": self.get_checkrun_text(err=err, out=out), + "text": self.get_check_run_text(err=err, out=out), } if rc: LOGGER.info(f"{self.log_prefix} Done building {_container_repository_and_tag}") @@ -1584,7 +1599,7 @@ def _run_install_python_module(self): output = { "title": "Python module installation", "summary": "", - "text": self.get_checkrun_text(err=err, out=out), + "text": self.get_check_run_text(err=err, out=out), } if rc: return self.set_python_module_install_success(output=output) @@ -1662,8 +1677,15 @@ def is_check_run_in_progress(self, check_run): return True return False - def set_check_run_status(self, check_run, status=None, conclusion=None, output=None): - kwargs = {"name": check_run, "head_sha": self.last_commit.sha} + def set_check_run_status( + self, + check_run: str, + status: str = "", + conclusion: str = "", + output: str = "", + ): + kwargs: Dict[str, str] = {"name": check_run, "head_sha": self.last_commit.sha} + if status: kwargs["status"] = status @@ -1673,7 +1695,7 @@ def set_check_run_status(self, check_run, status=None, conclusion=None, output=N if output: kwargs["output"] = output - msg = f"{self.log_prefix} Set {check_run} check to {status or conclusion}" + msg: str = f"{self.log_prefix} Set {check_run} check to {status or conclusion}" LOGGER.info(msg) try: @@ -1681,19 +1703,24 @@ def set_check_run_status(self, check_run, status=None, conclusion=None, output=N if conclusion == SUCCESS_STR: LOGGER.success(msg) + return + except Exception as ex: LOGGER.error(f"{self.log_prefix} Failed to set {check_run} check to {status or conclusion}, {ex}") kwargs["conclusion"] = FAILURE_STR + + LOGGER.error( + f"{self.log_prefix} Check run {check_run}, status: {FAILURE_STR}, output: {kwargs.get('output')}" + ) self.repository_by_github_app.create_check_run(**kwargs) - return f"Done setting check run status: {kwargs}" def _run_in_container( self, command: str, env: str = "", is_merged: bool = False, - checkout: Optional[str] = None, - tag_name: Optional[str] = None, + checkout: str = "", + tag_name: str = "", ) -> Tuple[int, str, str]: podman_base_cmd: str = ( "podman run --network=host --privileged -v /tmp/containers:/var/lib/containers/:Z " @@ -1727,16 +1754,17 @@ def _run_in_container( else: if not self.pull_request: LOGGER.error(f"{self.log_prefix} [func:_run_in_container] No pull request found") - return (False, "", "") + return False, "", "" clone_base_cmd += f" && git checkout origin/pr/{self.pull_request.number}" # final podman command podman_base_cmd += f" '{clone_base_cmd} && {command}'" return run_command(command=podman_base_cmd, log_prefix=self.log_prefix) - def get_checkrun_text(self, err, out): + @staticmethod + def get_check_run_text(err, out): total_len = len(err) + len(out) - if total_len > 65534: # Github limit is 65535 characters + if total_len > 65534: # GitHub limit is 65535 characters return f"```\n{err}\n\n{out}\n```"[:65534] else: return f"```\n{err}\n\n{out}\n```" @@ -1791,8 +1819,8 @@ def get_branch_required_status_checks(self): return [] pull_request_branch = self.repository.get_branch(self.pull_request_branch) - branch_protaction = pull_request_branch.get_protection() - return branch_protaction.required_status_checks.contexts + branch_protection = pull_request_branch.get_protection() + return branch_protection.required_status_checks.contexts def get_all_required_status_checks(self): all_required_status_checks = [] diff --git a/webhook_server_container/utils/github_repository_settings.py b/webhook_server_container/utils/github_repository_settings.py index 88c0e2b8..ac890d48 100644 --- a/webhook_server_container/utils/github_repository_settings.py +++ b/webhook_server_container/utils/github_repository_settings.py @@ -1,7 +1,8 @@ import contextlib import os -from concurrent.futures import ThreadPoolExecutor, as_completed +from concurrent.futures import Future, ThreadPoolExecutor, as_completed from copy import deepcopy +from typing import List from github import GithubIntegration, Auth from github.GithubException import UnknownObjectException @@ -25,7 +26,10 @@ ) -LOGGER = get_logger(name="github-repository-settings", filename=os.environ.get("WEBHOOK_SERVER_LOG_FILE")) +LOGGER = get_logger( + name="github-repository-settings", + filename=os.environ.get("WEBHOOK_SERVER_LOG_FILE"), +) @ignore_exceptions(logger=LOGGER) @@ -150,7 +154,16 @@ def set_repositories_settings(config, github_api): futures = [] with ThreadPoolExecutor() as executor: for _, data in config_data["repositories"].items(): - futures.append(executor.submit(set_repository, data, github_api, default_status_checks)) + futures.append( + executor.submit( + set_repository, + **{ + "data": data, + "github_api": github_api, + "default_status_checks": default_status_checks, + }, + ) + ) for result in as_completed(futures): if result.exception(): @@ -175,39 +188,53 @@ def set_repository(data, github_api, default_status_checks): LOGGER.warning(f"{repository}: Repository is private, skipping setting branch settings") return - for branch_name, status_checks in protected_branches.items(): - LOGGER.info(f"{repository}: Getting branch {branch_name}") - branch = get_branch_sampler(repo=repo, branch_name=branch_name) - if not branch: - LOGGER.error(f"{repository}: Failed to get branch {branch_name}") - continue + futures: List[Future] = [] + + with ThreadPoolExecutor() as executor: + for branch_name, status_checks in protected_branches.items(): + LOGGER.info(f"{repository}: Getting branch {branch_name}") + branch = get_branch_sampler(repo=repo, branch_name=branch_name) + if not branch: + LOGGER.error(f"{repository}: Failed to get branch {branch_name}") + continue + + _default_status_checks = deepcopy(default_status_checks) + ( + include_status_checks, + exclude_status_checks, + ) = get_user_configures_status_checks(status_checks=status_checks) + + required_status_checks = include_status_checks or get_required_status_checks( + repo=repo, + data=data, + default_status_checks=_default_status_checks, + exclude_status_checks=exclude_status_checks, + ) - _default_status_checks = deepcopy(default_status_checks) - ( - include_status_checks, - exclude_status_checks, - ) = get_user_configures_status_checks(status_checks=status_checks) - - required_status_checks = include_status_checks or get_required_status_checks( - repo=repo, - data=data, - default_status_checks=_default_status_checks, - exclude_status_checks=exclude_status_checks, - ) + futures.append( + executor.submit( + set_branch_protection, + **{ + "branch": branch, + "repository": repo, + "required_status_checks": required_status_checks, + "github_api": github_api, + }, + ) + ) + + for result in as_completed(futures): + if result.exception(): + LOGGER.error(result.exception()) + LOGGER.info(result.result()) - set_branch_protection( - branch=branch, - repository=repo, - required_status_checks=required_status_checks, - github_api=github_api, - ) except UnknownObjectException: LOGGER.error(f"{repository}: Failed to get repository settings") return f"{repository}: Setting repository settings is done" -def set_all_in_progress_check_runs_to_queued(config, github_api): +def set_all_in_progress_check_runs_to_queued(config_, github_api): check_runs = ( PYTHON_MODULE_INSTALL_STR, CAN_BE_MERGED_STR, @@ -217,14 +244,16 @@ def set_all_in_progress_check_runs_to_queued(config, github_api): ) futures = [] with ThreadPoolExecutor() as executor: - for _, data in config.data["repositories"].items(): + for _, data in config_.data["repositories"].items(): futures.append( executor.submit( set_repository_check_runs_to_queued, - config, - data, - github_api, - check_runs, + **{ + "config_": config_, + "data": data, + "github_api": github_api, + "check_runs": check_runs, + }, ) ) @@ -234,9 +263,9 @@ def set_all_in_progress_check_runs_to_queued(config, github_api): LOGGER.info(result.result()) -def set_repository_check_runs_to_queued(config, data, github_api, check_runs): +def set_repository_check_runs_to_queued(config_, data, github_api, check_runs): repository = data["name"] - repository_app_api = get_repository_github_app_api(config=config, repository=repository) + repository_app_api = get_repository_github_app_api(config_=config_, repository=repository) if not repository_app_api: return @@ -257,12 +286,12 @@ def set_repository_check_runs_to_queued(config, data, github_api, check_runs): @ignore_exceptions(logger=LOGGER) -def get_repository_github_app_api(config, repository): +def get_repository_github_app_api(config_, repository): LOGGER.info("Getting repositories GitHub app API") - with open(os.path.join(config.data_dir, "webhook-server.private-key.pem")) as fd: + with open(os.path.join(config_.data_dir, "webhook-server.private-key.pem")) as fd: private_key = fd.read() - github_app_id = config.data["github-app-id"] + github_app_id = config_.data["github-app-id"] auth = Auth.AppAuth(app_id=github_app_id, private_key=private_key) app_instance = GithubIntegration(auth=auth) owner, repo = repository.split("/") @@ -280,6 +309,6 @@ def get_repository_github_app_api(config, repository): api, _ = get_api_with_highest_rate_limit(config=config) set_repositories_settings(config=config, github_api=api) set_all_in_progress_check_runs_to_queued( - config=config, + config_=config, github_api=api, ) diff --git a/webhook_server_container/utils/webhook.py b/webhook_server_container/utils/webhook.py index a286352e..f611aadb 100644 --- a/webhook_server_container/utils/webhook.py +++ b/webhook_server_container/utils/webhook.py @@ -1,10 +1,14 @@ from concurrent.futures import ThreadPoolExecutor, as_completed import os +from github import Github from simple_logger.logger import get_logger from webhook_server_container.libs.config import Config -from webhook_server_container.utils.helpers import get_api_with_highest_rate_limit, get_github_repo_api +from webhook_server_container.utils.helpers import ( + get_api_with_highest_rate_limit, + get_github_repo_api, +) from pyhelper_utils.general import ignore_exceptions @@ -37,14 +41,19 @@ def process_github_webhook(data, github_api, webhook_ip): return f"{repository}: Create webhook is done" -def create_webhook(config, github_api): +def create_webhook(config_: Config, github_api: Github) -> None: LOGGER.info("Preparing webhook configuration") - webhook_ip = config.data["webhook_ip"] + webhook_ip = config_.data["webhook_ip"] futures = [] with ThreadPoolExecutor() as executor: - for repo, data in config.data["repositories"].items(): - futures.append(executor.submit(process_github_webhook, data, github_api, webhook_ip)) + for repo, data in config_.data["repositories"].items(): + futures.append( + executor.submit( + process_github_webhook, + **{"data": data, "github_api": github_api, "webhook_ip": webhook_ip}, + ) + ) for result in as_completed(futures): if result.exception(): @@ -55,4 +64,4 @@ def create_webhook(config, github_api): if __name__ == "__main__": config = Config() api, _ = get_api_with_highest_rate_limit(config=config) - create_webhook(config=config, github_api=api) + create_webhook(config_=config, github_api=api)