diff --git a/Dockerfile b/Dockerfile index 3d47d0a2..c6e2fa64 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,7 @@ RUN mkdir -p $BIN_DIR \ && mkdir -p $DATA_DIR \ && mkdir -p $DATA_DIR/logs -COPY entrypoint.sh pyproject.toml uv.lock README.md $APP_DIR/ +COPY gunicorn.conf.py pyproject.toml uv.lock README.md $APP_DIR/ COPY webhook_server $APP_DIR/webhook_server/ RUN usermod --add-subuids 100000-165535 --add-subgids 100000-165535 $USERNAME \ @@ -66,4 +66,4 @@ RUN uv sync HEALTHCHECK CMD curl --fail http://127.0.0.1:5000/webhook_server/healthcheck || exit 1 -ENTRYPOINT ["./entrypoint.sh"] +ENTRYPOINT ["uv", "run", "gunicorn", "webhook_server.app:FASTAPI_APP", "-c", "./gunicorn.conf.py"] diff --git a/docker-compose-example.yaml b/docker-compose-example.yaml index 17c14907..e7bc0d4c 100644 --- a/docker-compose-example.yaml +++ b/docker-compose-example.yaml @@ -8,8 +8,9 @@ services: - PUID=1000 - PGID=1000 - TZ=Asia/Jerusalem - - DEVELOPMENT=false # Set to true when developing. - UVICORN_MAX_WORKERS=50 # Defaults to 10 if not set and running in production + - WEBHOOK_SERVER_IP_BIND=0.0.0.0 # IP to listen + - WEBHOOK_SERVER_PORT=5000 # Port to listen ports: - "5000:5000" privileged: true diff --git a/entrypoint.sh b/entrypoint.sh deleted file mode 100755 index 8b4a077b..00000000 --- a/entrypoint.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -SERVER_RUN_CMD="uv run uvicorn webhook_server.app:FASTAPI_APP --host 0.0.0.0 --port 5000 " -UVICORN_WORKERS="${UVICORN_MAX_WORKERS:=10}" - -set -ep - -uv run webhook_server/utils/github_repository_and_webhook_settings.py - -if [[ -z $DEVELOPMENT ]]; then - eval "${SERVER_RUN_CMD} --workers ${UVICORN_WORKERS}" -else - eval "${SERVER_RUN_CMD} --reload" -fi diff --git a/gunicorn.conf.py b/gunicorn.conf.py new file mode 100644 index 00000000..0a574645 --- /dev/null +++ b/gunicorn.conf.py @@ -0,0 +1,8 @@ +import os + +bind = f"{os.environ.get('WEBHOOK_SERVER_IP_BIND', '0.0.0.0')}:{os.environ.get('WEBHOOK_SERVER_PORT', 5000)}" +worker_class = "uvicorn.workers.UvicornWorker" +workers = int(os.environ.get("UVICORN_WORKERS", 10)) +on_starting = "webhook_server.app.on_starting" +accesslog = "-" +timeout = 0 # Disable workers timeout, some operation can take long time. diff --git a/pyproject.toml b/pyproject.toml index a7daa3f0..c9a00457 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [tool.coverage.run] -omit = [ "tests/*" ] +omit = ["tests/*"] [tool.coverage.report] fail_under = 35 @@ -14,8 +14,8 @@ line-length = 120 fix = true output-format = "grouped" - [tool.ruff.format] - exclude = [ ".git", ".venv", ".mypy_cache", ".tox", "__pycache__" ] +[tool.ruff.format] +exclude = [".git", ".venv", ".mypy_cache", ".tox", "__pycache__"] [tool.mypy] check_untyped_defs = true @@ -31,10 +31,10 @@ warn_unused_configs = true warn_redundant_casts = true [tool.hatch.build.targets.wheel] -packages = [ "webhook_server" ] +packages = ["webhook_server"] [tool.uv] -dev-dependencies = [ "ipdb>=0.13.13", "ipython>=8.12.3" ] +dev-dependencies = ["ipdb>=0.13.13", "ipython>=8.12.3"] [project] name = "github-webhook-server" @@ -45,7 +45,7 @@ readme = "README.md" license = "Apache-2.0" classifiers = [ "Programming Language :: Python :: 3", - "Operating System :: OS Independent" + "Operating System :: OS Independent", ] dependencies = [ "build>=1.2.2.post1", @@ -65,23 +65,24 @@ dependencies = [ "string-color>=1.2.3", "timeout-sampler>=0.0.46", "uvicorn>=0.31.0", - "uwsgi>=2.0.27" + "uvicorn-worker>=0.3.0", + "gunicorn>=23.0.0", ] - [[project.authors]] - name = "Meni Yakove" - email = " myakove@gmail.com" +[[project.authors]] +name = "Meni Yakove" +email = " myakove@gmail.com" - [[project.authors]] - name = "Ruth Netser" - email = "ruth.netser@gmail.com" +[[project.authors]] +name = "Ruth Netser" +email = "ruth.netser@gmail.com" - [project.urls] - homepage = "https://github.com/myakove/github-webhook-server" - repository = "https://github.com/myakove/github-webhook-server" - Download = "https://quay.io/repository/myakove/github-webhook-server" - "Bug Tracker" = "https://github.com/myakove/github-webhook-server/issues" +[project.urls] +homepage = "https://github.com/myakove/github-webhook-server" +repository = "https://github.com/myakove/github-webhook-server" +Download = "https://quay.io/repository/myakove/github-webhook-server" +"Bug Tracker" = "https://github.com/myakove/github-webhook-server/issues" [build-system] -requires = [ "hatchling" ] +requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/uv.lock b/uv.lock index 5fd4c380..f8124b9c 100644 --- a/uv.lock +++ b/uv.lock @@ -458,6 +458,7 @@ dependencies = [ { name = "colorama" }, { name = "colorlog" }, { name = "fastapi" }, + { name = "gunicorn" }, { name = "pygithub" }, { name = "pyhelper-utils" }, { name = "pytest" }, @@ -471,6 +472,7 @@ dependencies = [ { name = "string-color" }, { name = "timeout-sampler" }, { name = "uvicorn" }, + { name = "uvicorn-worker" }, { name = "uwsgi" }, ] @@ -487,6 +489,7 @@ requires-dist = [ { name = "colorama", specifier = ">=0.4.6" }, { name = "colorlog", specifier = ">=6.8.2" }, { name = "fastapi", specifier = ">=0.115.0" }, + { name = "gunicorn", specifier = ">=23.0.0" }, { name = "pygithub", specifier = ">=2.4.0" }, { name = "pyhelper-utils", specifier = ">=0.0.42" }, { name = "pytest", specifier = ">=8.3.3" }, @@ -500,6 +503,7 @@ requires-dist = [ { name = "string-color", specifier = ">=1.2.3" }, { name = "timeout-sampler", specifier = ">=0.0.46" }, { name = "uvicorn", specifier = ">=0.31.0" }, + { name = "uvicorn-worker", specifier = ">=0.3.0" }, { name = "uwsgi", specifier = ">=2.0.27" }, ] @@ -509,6 +513,18 @@ dev = [ { name = "ipython", specifier = ">=8.12.3" }, ] +[[package]] +name = "gunicorn" +version = "23.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/34/72/9614c465dc206155d93eff0ca20d42e1e35afc533971379482de953521a4/gunicorn-23.0.0.tar.gz", hash = "sha256:f014447a0101dc57e294f6c18ca6b40227a4c90e9bdb586042628030cba004ec", size = 375031 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/7d/6dac2a6e1eba33ee43f318edbed4ff29151a49b5d37f080aad1e6469bca4/gunicorn-23.0.0-py3-none-any.whl", hash = "sha256:ec400d38950de4dfd418cff8328b2c8faed0edb0d517d3394e457c317908ca4d", size = 85029 }, +] + [[package]] name = "h11" version = "0.14.0" @@ -1315,6 +1331,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b1/4b/4cef6ce21a2aaca9d852a6e84ef4f135d99fcd74fa75105e2fc0c8308acd/uvicorn-0.34.2-py3-none-any.whl", hash = "sha256:deb49af569084536d269fe0a6d67e3754f104cf03aba7c11c40f01aadf33c403", size = 62483 }, ] +[[package]] +name = "uvicorn-worker" +version = "0.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "gunicorn" }, + { name = "uvicorn" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/37/c0/b5df8c9a31b0516a47703a669902b362ca1e569fed4f3daa1d4299b28be0/uvicorn_worker-0.3.0.tar.gz", hash = "sha256:6baeab7b2162ea6b9612cbe149aa670a76090ad65a267ce8e27316ed13c7de7b", size = 9181 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/1f/4e5f8770c2cf4faa2c3ed3c19f9d4485ac9db0a6b029a7866921709bdc6c/uvicorn_worker-0.3.0-py3-none-any.whl", hash = "sha256:ef0fe8aad27b0290a9e602a256b03f5a5da3a9e5f942414ca587b645ec77dd52", size = 5346 }, +] + [[package]] name = "uwsgi" version = "2.0.29" diff --git a/webhook_server/app.py b/webhook_server/app.py index 44ffcc53..9cfb57c8 100644 --- a/webhook_server/app.py +++ b/webhook_server/app.py @@ -7,6 +7,7 @@ from fastapi import FastAPI, Request from webhook_server.libs.github_api import ProcessGithubWehook +from webhook_server.utils.github_repository_and_webhook_settings import repository_and_webhook_settings from webhook_server.utils.helpers import get_logger_with_params FASTAPI_APP: FastAPI = FastAPI(title="webhook-server") @@ -14,6 +15,10 @@ urllib3.disable_warnings() +def on_starting(server: Any) -> None: + repository_and_webhook_settings() + + @FASTAPI_APP.get(f"{APP_URL_ROOT_PATH}/healthcheck") def healthcheck() -> dict[str, Any]: return {"status": requests.codes.ok, "message": "Alive"} diff --git a/webhook_server/libs/github_api.py b/webhook_server/libs/github_api.py index d86a3507..058b3635 100644 --- a/webhook_server/libs/github_api.py +++ b/webhook_server/libs/github_api.py @@ -957,8 +957,7 @@ def _run_tox(self) -> None: return if self.is_check_run_in_progress(check_run=TOX_STR): - self.logger.debug(f"{self.log_prefix} Check run is in progress, not running {TOX_STR}.") - return + self.logger.debug(f"{self.log_prefix} Check run is in progress, re-running {TOX_STR}.") clone_repo_dir = f"{self.clone_repo_dir}-{uuid4()}" python_ver = f"--python={self.tox_python_version}" if self.tox_python_version else "" @@ -993,8 +992,7 @@ def _run_pre_commit(self) -> None: return if self.is_check_run_in_progress(check_run=PRE_COMMIT_STR): - self.logger.debug(f"{self.log_prefix} Check run is in progress, not running {PRE_COMMIT_STR}.") - return + self.logger.debug(f"{self.log_prefix} Check run is in progress, re-running {PRE_COMMIT_STR}.") clone_repo_dir = f"{self.clone_repo_dir}-{uuid4()}" cmd = f" uvx --directory {clone_repo_dir} {PRE_COMMIT_STR} run --all-files" @@ -1331,13 +1329,15 @@ def _run_build_container( if not self.build_and_push_container: return + if self.is_check_run_in_progress(check_run=BUILD_CONTAINER_STR): + self.logger.info(f"{self.log_prefix} Check run is in progress, re-running {BUILD_CONTAINER_STR}.") + clone_repo_dir = f"{self.clone_repo_dir}-{uuid4()}" pull_request = hasattr(self, "pull_request") if pull_request and set_check: if self.is_check_run_in_progress(check_run=BUILD_CONTAINER_STR) and not is_merged: - self.logger.info(f"{self.log_prefix} Check run is in progress, not running {BUILD_CONTAINER_STR}.") - return + self.logger.info(f"{self.log_prefix} Check run is in progress, re-running {BUILD_CONTAINER_STR}.") self.set_container_build_in_progress() @@ -1418,8 +1418,7 @@ def _run_install_python_module(self) -> None: return if self.is_check_run_in_progress(check_run=PYTHON_MODULE_INSTALL_STR): - self.logger.info(f"{self.log_prefix} Check run is in progress, not running {PYTHON_MODULE_INSTALL_STR}.") - return + self.logger.info(f"{self.log_prefix} Check run is in progress, re-running {PYTHON_MODULE_INSTALL_STR}.") clone_repo_dir = f"{self.clone_repo_dir}-{uuid4()}" self.logger.info(f"{self.log_prefix} Installing python module") @@ -2172,6 +2171,10 @@ def _run_conventional_title_check(self) -> None: "summary": "", "text": "", } + + if self.is_check_run_in_progress(check_run=CONVENTIONAL_TITLE_STR): + self.logger.info(f"{self.log_prefix} Check run is in progress, re-running {CONVENTIONAL_TITLE_STR}.") + self.set_conventional_title_in_progress() allowed_names = self.conventional_title.split(",") title = self.pull_request.title diff --git a/webhook_server/utils/github_repository_and_webhook_settings.py b/webhook_server/utils/github_repository_and_webhook_settings.py index fbbf22d6..5a8a5700 100644 --- a/webhook_server/utils/github_repository_and_webhook_settings.py +++ b/webhook_server/utils/github_repository_and_webhook_settings.py @@ -18,7 +18,7 @@ def get_repository_api(repository: str) -> tuple[str, github.Github | None, str] return repository, github_api, api_user -if __name__ == "__main__": +def repository_and_webhook_settings() -> None: logger = get_logger_with_params(name="github-repository-and-webhook-settings") config = Config() @@ -43,3 +43,7 @@ def get_repository_api(repository: str) -> tuple[str, github.Github | None, str] set_repositories_settings(config=config, apis_dict=apis_dict) set_all_in_progress_check_runs_to_queued(repo_config=config, apis_dict=apis_dict) create_webhook(config=config, apis_dict=apis_dict) + + +if __name__ == "__main__": + repository_and_webhook_settings()