Use uwsgi to run the app

Dockerfile

@@ -40,7 +40,7 @@ RUN python3.8 -m ensurepip \
   && python3.11 -m pip install tox \
   && python3.12 -m pip install tox
 
-COPY pyproject.toml poetry.lock README.md $APP_DIR/
+COPY entrypoint.sh pyproject.toml poetry.lock README.md $APP_DIR/
 COPY webhook_server_container $APP_DIR/webhook_server_container/
 
 WORKDIR $APP_DIR
@@ -51,4 +51,4 @@ RUN poetry config cache-dir $APP_DIR \
   && poetry install
 
 HEALTHCHECK CMD curl --fail http://127.0.0.1:5000/webhook_server/healthcheck || exit 1
-ENTRYPOINT ["poetry", "run", "python3", "webhook_server_container/app.py"]
+ENTRYPOINT ["./entrypoint.sh"]

docker-compose-example.yaml

@@ -1,5 +1,3 @@
-# All ${} taken from .env file, create .env file in root of project.
-version: "3.8"
 services:
   github-webhook-server:
     container_name: github-webhook-server
@@ -12,6 +10,7 @@ services:
       - PUID=1000
       - PGID=1000
       - TZ=Asia/Jerusalem
+      - DEVELOPMENT=false # Set to true when developing.
     ports:
       - "5000:5000"
     privileged: true

entrypoint.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -ep
+
+poetry run python webhook_server_container/utils/github_repository_settings.py
+poetry run python webhook_server_container/utils/webhook.py
+
+if [[ -z $DEVELOPMENT ]]; then
+	poetry run uwsgi --disable-logging --post-buffering --master --enable-threads --http 0.0.0.0:5000 --wsgi-file webhook_server_container/app.py --callable FLASK_APP --processes 4 --threads 2
+else
+	poetry run python webhook_server_container/app.py
+fi

pyproject.toml

@@ -42,6 +42,7 @@ timeout-sampler = "^0.0.32"
 requests = "^2.31.0"
 jira = "^3.8.0"
 pyhelper-utils = "^0.0.22"
+uwsgi = "^2.0.26"
 
 [tool.poetry.group.dev.dependencies]
 ipdb = "^0.13.13"

webhook_server_container/app.py

@@ -2,53 +2,18 @@
 
 import urllib3
 from flask import request
-from github import Auth, GithubIntegration
 
-from webhook_server_container.libs.config import Config
 from webhook_server_container.libs.github_api import GitHubApi
 from webhook_server_container.utils.constants import (
     APP_ROOT_PATH,
     FLASK_APP,
 )
-from webhook_server_container.utils.github_repository_settings import (
-    set_all_in_progress_check_runs_to_queued,
-    set_repositories_settings,
-)
-from pyhelper_utils.general import ignore_exceptions
-from webhook_server_container.utils.helpers import (
-    get_api_with_highest_rate_limit,
-)
-from webhook_server_container.utils.webhook import create_webhook
 
 REPOSITORIES_APP_API = {}
 MISSING_APP_REPOSITORIES = []
-
 urllib3.disable_warnings()
 
 
-@ignore_exceptions(logger=FLASK_APP.logger)
-def get_repositories_github_app_api(config):
-    FLASK_APP.logger.info("Getting repositories GitHub app API")
-    with open(os.path.join(config.data_dir, "webhook-server.private-key.pem")) as fd:
-        private_key = fd.read()
-
-    github_app_id = config.data["github-app-id"]
-    auth = Auth.AppAuth(app_id=github_app_id, private_key=private_key)
-    for installation in GithubIntegration(auth=auth).get_installations():
-        for repo in installation.get_repos():
-            FLASK_APP.logger.info(f"Getting repository {repo.full_name} GitHub app API")
-            REPOSITORIES_APP_API[repo.full_name] = installation.get_github_for_installation()
-
-    for data in config.data["repositories"].values():
-        full_name = data["name"]
-        if not REPOSITORIES_APP_API.get(full_name):
-            FLASK_APP.logger.error(
-                f"Repository {full_name} not found by manage-repositories-app, "
-                f"make sure the app installed (https://github.com/apps/manage-repositories-app)"
-            )
-            MISSING_APP_REPOSITORIES.append(full_name)
-
-
 @FLASK_APP.route(f"{APP_ROOT_PATH}/healthcheck")
 def healthcheck():
     return "alive"
@@ -64,11 +29,7 @@ def process_webhook():
         return process_failed_msg
 
     try:
-        api = GitHubApi(
-            hook_data=hook_data,
-            repositories_app_api=REPOSITORIES_APP_API,
-            missing_app_repositories=MISSING_APP_REPOSITORIES,
-        )
+        api = GitHubApi(hook_data=hook_data)
     except Exception as ex:
         FLASK_APP.logger.error(f"Failed to initialized GitHubApi instance: {ex}")
         return process_failed_msg
@@ -85,22 +46,12 @@ def process_webhook():
 
 
 def main():
-    config = Config()
-    api, _ = get_api_with_highest_rate_limit(config=config)
-    get_repositories_github_app_api(config=config)
-    set_repositories_settings(config=config, github_api=api)
-    set_all_in_progress_check_runs_to_queued(
-        config=config,
-        repositories_app_api=REPOSITORIES_APP_API,
-        missing_app_repositories=MISSING_APP_REPOSITORIES,
-        github_api=api,
-    )
-    create_webhook(config=config, github_api=api)
     FLASK_APP.logger.info(f"Starting {FLASK_APP.name} app")
     FLASK_APP.run(
-        port=int(os.environ.get("WEBHOOK_SERVER_PORT", 5000)),
+        port=5000,
         host="0.0.0.0",
-        use_reloader=True if os.environ.get("WEBHOOK_SERVER_USE_RELOAD") else False,
+        use_reloader=bool(os.getenv("WEBHOOK_SERVER_USE_RELOAD", False)),
+        debug=bool(os.getenv("WEBHOOK_SERVER_USE_DEBUG", False)),
     )
 
 

webhook_server_container/libs/github_api.py

@@ -52,6 +52,7 @@
 )
 from pyhelper_utils.general import ignore_exceptions
 from webhook_server_container.utils.dockerhub_rate_limit import DockerHub
+from webhook_server_container.utils.github_repository_settings import get_repository_github_app_api
 from webhook_server_container.utils.helpers import (
     get_api_with_highest_rate_limit,
     extract_key_from_dict,
@@ -67,12 +68,10 @@ class RepositoryNotFoundError(Exception):
 
 
 class GitHubApi:
-    def __init__(self, hook_data, repositories_app_api, missing_app_repositories):
+    def __init__(self, hook_data):
         self.app = FLASK_APP
         self.hook_data = hook_data
         self.repository_name = hook_data["repository"]["name"]
-        self.repositories_app_api = repositories_app_api
-        self.missing_app_repositories = missing_app_repositories
         self.log_prefix_with_color = None
         self.pull_request = None
         self.parent_committer = None
@@ -107,7 +106,13 @@ def __init__(self, hook_data, repositories_app_api, missing_app_repositories):
         self._set_log_prefix_color()
         # self.log_repository_features()
 
-        self.github_app_api = self.get_github_app_api()
+        self.github_app_api = get_repository_github_app_api(config=self.config, repository=self.repository_full_name)
+        if not self.github_app_api:
+            FLASK_APP.logger.error(
+                f"Repository {self.repository_full_name} not found by manage-repositories-app, "
+                f"make sure the app installed (https://github.com/apps/manage-repositories-app)"
+            )
+            return
 
         self.github_api, self.token = get_api_with_highest_rate_limit(
             config=self.config, repository_name=self.repository_name
@@ -204,14 +209,6 @@ def prepare_retest_wellcome_msg(self):
 
         return " * This repository does not support retest actions" if not retest_msg else retest_msg
 
-    def get_github_app_api(self):
-        if self.repository_full_name in self.missing_app_repositories:
-            raise RepositoryNotFoundError(
-                f"Repository {self.repository_full_name} not found by manage-repositories-app, "
-                f"make sure the app installed (https://github.com/apps/manage-repositories-app)"
-            )
-        return self.repositories_app_api[self.repository_full_name]
-
     def add_api_users_to_auto_verified_and_merged_users(self):
         apis_and_tokens = get_apis_and_tokes_from_config(config=self.config, repository_name=self.repository_name)
         self.auto_verified_and_merged_users.extend([_api[0].get_user().login for _api in apis_and_tokens])

webhook_server_container/utils/github_repository_settings.py

@@ -3,8 +3,10 @@
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from copy import deepcopy
 
+from github import GithubIntegration, Auth
 from github.GithubException import UnknownObjectException
 
+from webhook_server_container.libs.config import Config
 from webhook_server_container.utils.constants import (
     BUILD_CONTAINER_STR,
     CAN_BE_MERGED_STR,
@@ -18,6 +20,7 @@
 )
 from pyhelper_utils.general import ignore_exceptions
 from webhook_server_container.utils.helpers import (
+    get_api_with_highest_rate_limit,
     get_github_repo_api,
 )
 
@@ -211,7 +214,7 @@ def set_repository(data, github_api, default_status_checks):
     return f"{repository}: Setting repository settings is done"
 
 
-def set_all_in_progress_check_runs_to_queued(config, repositories_app_api, missing_app_repositories, github_api):
+def set_all_in_progress_check_runs_to_queued(config, github_api):
     check_runs = (
         PYTHON_MODULE_INSTALL_STR,
         CAN_BE_MERGED_STR,
@@ -225,9 +228,8 @@ def set_all_in_progress_check_runs_to_queued(config, repositories_app_api, missi
             futures.append(
                 executor.submit(
                     set_repository_check_runs_to_queued,
+                    config,
                     data,
-                    missing_app_repositories,
-                    repositories_app_api,
                     github_api,
                     check_runs,
                 )
@@ -239,12 +241,13 @@ def set_all_in_progress_check_runs_to_queued(config, repositories_app_api, missi
         FLASK_APP.logger.info(result.result())
 
 
-def set_repository_check_runs_to_queued(data, missing_app_repositories, repositories_app_api, github_api, check_runs):
+def set_repository_check_runs_to_queued(config, data, github_api, check_runs):
     repository = data["name"]
-    if repository in missing_app_repositories:
+    repository_app_api = get_repository_github_app_api(config=config, repository=repository)
+    if not repository_app_api:
         return
 
-    app_api = get_github_repo_api(github_api=repositories_app_api[repository], repository=repository)
+    app_api = get_github_repo_api(github_api=repository_app_api, repository=repository)
     repo = get_github_repo_api(github_api=github_api, repository=repository)
     FLASK_APP.logger.info(f"{repository}: Set all {IN_PROGRESS_STR} check runs to {QUEUED_STR}")
     for pull_request in repo.get_pulls(state="open"):
@@ -258,3 +261,32 @@ def set_repository_check_runs_to_queued(data, missing_app_repositories, reposito
                 app_api.create_check_run(name=check_run.name, head_sha=last_commit.sha, status=QUEUED_STR)
 
     return f"{repository}: Set check run status to {QUEUED_STR} is done"
+
+
+@ignore_exceptions(logger=FLASK_APP.logger)
+def get_repository_github_app_api(config, repository):
+    FLASK_APP.logger.info("Getting repositories GitHub app API")
+    with open(os.path.join(config.data_dir, "webhook-server.private-key.pem")) as fd:
+        private_key = fd.read()
+
+    github_app_id = config.data["github-app-id"]
+    auth = Auth.AppAuth(app_id=github_app_id, private_key=private_key)
+    app_instance = GithubIntegration(auth=auth)
+    owner, repo = repository.split("/")
+    try:
+        return app_instance.get_repo_installation(owner=owner, repo=repo).get_github_for_installation()
+    except UnknownObjectException:
+        FLASK_APP.logger.error(
+            f"Repository {repository} not found by manage-repositories-app, "
+            f"make sure the app installed (https://github.com/apps/manage-repositories-app)"
+        )
+
+
+if __name__ == "__main__":
+    config = Config()
+    api, _ = get_api_with_highest_rate_limit(config=config)
+    set_repositories_settings(config=config, github_api=api)
+    set_all_in_progress_check_runs_to_queued(
+        config=config,
+        github_api=api,
+    )

webhook_server_container/utils/webhook.py

@@ -1,8 +1,9 @@
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
 
+from webhook_server_container.libs.config import Config
 from webhook_server_container.utils.constants import FLASK_APP
-from webhook_server_container.utils.helpers import get_github_repo_api
+from webhook_server_container.utils.helpers import get_api_with_highest_rate_limit, get_github_repo_api
 from pyhelper_utils.general import ignore_exceptions
 
 
@@ -47,3 +48,9 @@ def create_webhook(config, github_api):
         if result.exception():
             FLASK_APP.logger.error(result.exception())
         FLASK_APP.logger.info(result.result())
+
+
+if __name__ == "__main__":
+    config = Config()
+    api, _ = get_api_with_highest_rate_limit(config=config)
+    create_webhook(config=config, github_api=api)