From 5a4e9c0f214cdde5f1aae9941311e89e99ae0d47 Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Sun, 13 Oct 2024 17:10:27 +0300 Subject: [PATCH 01/17] dockerfile: Do not run as root --- Dockerfile | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index 851a716f..dac4e5c7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,16 @@ FROM quay.io/podman/stable:latest + EXPOSE 5000 +ENV HOME_DIR=/home/webhook_server +ENV BIN_DIR="$HOME_DIR/.local/bin" +ENV UV_INSTALL_DIR="$HOME_DIR/.local" +ENV PATH="$PATH:$BIN_DIR" +ENV DATA_DIR=$HOME_DIR/data +ENV APP_DIR=$HOME_DIR/app + +RUN useradd -m -d $HOME_DIR webhook_server -s /bin/bash + RUN dnf -y install dnf-plugins-core \ && dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo \ && dnf -y update \ @@ -25,15 +35,19 @@ RUN dnf -y install dnf-plugins-core \ && dnf clean all \ && rm -rf /var/cache /var/log/dnf* /var/log/yum.* -ENV USER_BIN_DIR="/root/.local/bin" -ENV UV_INSTALL_DIR="/root/.local" -ENV PATH="$PATH:$USER_BIN_DIR" +RUN ln -s /usr/bin/python3 /usr/bin/python + +COPY entrypoint.sh pyproject.toml uv.lock README.md $APP_DIR/ +COPY webhook_server_container $APP_DIR/webhook_server_container/ -ENV DATA_DIR=/webhook_server -ENV APP_DIR=/github-webhook-server +RUN chown -R webhook_server:webhook_server $APP_DIR -RUN mkdir -p $USER_BIN_DIR \ +USER webhook_server +WORKDIR $HOME_DIR + +RUN mkdir -p $BIN_DIR \ && mkdir -p $DATA_DIR \ + && mkdir -p $APP_DIR \ && mkdir -p $DATA_DIR/logs \ && mkdir -p /tmp/containers @@ -43,13 +57,11 @@ RUN curl -sSL https://astral.sh/uv/install.sh -o /tmp/uv-installer.sh \ && rm /tmp/uv-installer.sh RUN set -x \ - && curl https://mirror.openshift.com/pub/openshift-v4/clients/rosa/latest/rosa-linux.tar.gz --output /tmp/rosa-linux.tar.gz \ - && tar xvf /tmp/rosa-linux.tar.gz --no-same-owner \ - && mv rosa $USER_BIN_DIR/rosa \ - && chmod +x $USER_BIN_DIR/rosa \ - && rm -rf /tmp/rosa-linux.tar.gz - -RUN ln -s /usr/bin/python3 /usr/bin/python + && curl https://mirror.openshift.com/pub/openshift-v4/clients/rosa/latest/rosa-linux.tar.gz --output $BIN_DIR/rosa-linux.tar.gz \ + && tar xvf $BIN_DIR/rosa-linux.tar.gz \ + && mv rosa $BIN_DIR/rosa \ + && chmod +x $BIN_DIR/rosa \ + && rm -rf $BIN_DIR/rosa-linux.tar.gz RUN python -m pip install --no-cache-dir pip --upgrade \ && python -m pip install --no-cache-dir poetry tox twine pre-commit @@ -65,10 +77,6 @@ RUN python3.8 -m ensurepip \ && python3.11 -m pip install tox \ && python3.12 -m pip install tox -COPY entrypoint.sh pyproject.toml uv.lock README.md $APP_DIR/ -COPY webhook_server_container $APP_DIR/webhook_server_container/ - -WORKDIR $APP_DIR HEALTHCHECK CMD curl --fail http://127.0.0.1:5000/webhook_server/healthcheck || exit 1 ENTRYPOINT ["./entrypoint.sh"] From 744295287cd90d55d2c34e782b054925dae9423c Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Sun, 13 Oct 2024 18:46:53 +0300 Subject: [PATCH 02/17] dockerfile: Do not run as root --- .example_env | 1 - Dockerfile | 30 +++++++++++++------------ docker-compose-example.yaml | 3 +-- webhook_server_container/app.py | 6 ++--- webhook_server_container/libs/config.py | 2 +- 5 files changed, 21 insertions(+), 21 deletions(-) delete mode 100644 .example_env diff --git a/.example_env b/.example_env deleted file mode 100644 index fc752ed5..00000000 --- a/.example_env +++ /dev/null @@ -1 +0,0 @@ -NGROK_AUTHTOKEN= diff --git a/Dockerfile b/Dockerfile index dac4e5c7..b6d5d888 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,14 +2,13 @@ FROM quay.io/podman/stable:latest EXPOSE 5000 -ENV HOME_DIR=/home/webhook_server +ENV USERNAME="podman" +ENV HOME_DIR="/home/$USERNAME" ENV BIN_DIR="$HOME_DIR/.local/bin" ENV UV_INSTALL_DIR="$HOME_DIR/.local" ENV PATH="$PATH:$BIN_DIR" -ENV DATA_DIR=$HOME_DIR/data -ENV APP_DIR=$HOME_DIR/app - -RUN useradd -m -d $HOME_DIR webhook_server -s /bin/bash +ENV DATA_DIR="$HOME_DIR/data" +ENV APP_DIR="$HOME_DIR/github-webhook-server" RUN dnf -y install dnf-plugins-core \ && dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo \ @@ -37,20 +36,19 @@ RUN dnf -y install dnf-plugins-core \ RUN ln -s /usr/bin/python3 /usr/bin/python -COPY entrypoint.sh pyproject.toml uv.lock README.md $APP_DIR/ -COPY webhook_server_container $APP_DIR/webhook_server_container/ - -RUN chown -R webhook_server:webhook_server $APP_DIR - -USER webhook_server -WORKDIR $HOME_DIR - RUN mkdir -p $BIN_DIR \ + RUN mkdir -p $APP_DIR \ && mkdir -p $DATA_DIR \ - && mkdir -p $APP_DIR \ && mkdir -p $DATA_DIR/logs \ && mkdir -p /tmp/containers +COPY entrypoint.sh pyproject.toml uv.lock README.md $APP_DIR/ +COPY webhook_server_container $APP_DIR/webhook_server_container/ + +RUN chown -R $USERNAME:$USERNAME $HOME_DIR +USER $USERNAME +WORKDIR $HOME_DIR + # Download the latest uv installer RUN curl -sSL https://astral.sh/uv/install.sh -o /tmp/uv-installer.sh \ && sh /tmp/uv-installer.sh \ @@ -77,6 +75,10 @@ RUN python3.8 -m ensurepip \ && python3.11 -m pip install tox \ && python3.12 -m pip install tox +WORKDIR $APP_DIR + +RUN uv sync HEALTHCHECK CMD curl --fail http://127.0.0.1:5000/webhook_server/healthcheck || exit 1 + ENTRYPOINT ["./entrypoint.sh"] diff --git a/docker-compose-example.yaml b/docker-compose-example.yaml index 566bc158..63643b16 100644 --- a/docker-compose-example.yaml +++ b/docker-compose-example.yaml @@ -3,8 +3,7 @@ services: container_name: github-webhook-server build: quay.io/myakove/github-webhook-server volumes: - - "./webhook_server_data_dir:/webhook_server:Z" # Should include config.yaml and webhook-server.private-key.pem - - "./webhook_server_containers:/containers:Z" # Optional, to cache podman pull containers + - "./webhook_server_data_dir:/home/podman/data:Z" # Should include config.yaml and webhook-server.private-key.pem environment: - PUID=1000 - PGID=1000 diff --git a/webhook_server_container/app.py b/webhook_server_container/app.py index a1e09e8c..f12b2368 100644 --- a/webhook_server_container/app.py +++ b/webhook_server_container/app.py @@ -10,16 +10,16 @@ from webhook_server_container.utils.helpers import get_logger_with_params FASTAPI_APP: FastAPI = FastAPI(title="webhook-server") -APP_ROOT_PATH: str = "/webhook_server" +APP_URL_ROOT_PATH: str = "/webhook_server" urllib3.disable_warnings() -@FASTAPI_APP.get(f"{APP_ROOT_PATH}/healthcheck") +@FASTAPI_APP.get(f"{APP_URL_ROOT_PATH}/healthcheck") def healthcheck() -> Dict[str, Any]: return {"status": requests.codes.ok, "message": "Alive"} -@FASTAPI_APP.post(APP_ROOT_PATH) +@FASTAPI_APP.post(APP_URL_ROOT_PATH) async def process_webhook(request: Request) -> Dict[str, Any]: logger_name: str = "main" logger = get_logger_with_params(name=logger_name) diff --git a/webhook_server_container/libs/config.py b/webhook_server_container/libs/config.py index f73c4b79..1bbb489e 100644 --- a/webhook_server_container/libs/config.py +++ b/webhook_server_container/libs/config.py @@ -6,7 +6,7 @@ class Config: def __init__(self) -> None: - self.data_dir: str = os.environ.get("WEBHOOK_SERVER_DATA_DIR", "/webhook_server") + self.data_dir: str = os.environ.get("WEBHOOK_SERVER_DATA_DIR", "/home/podman/data") self.config_path: str = os.path.join(self.data_dir, "config.yaml") self.exists() From 0219bbf535d55a48418381278a8ac7f9784023e5 Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Sun, 13 Oct 2024 19:18:27 +0300 Subject: [PATCH 03/17] Fix podman command --- Dockerfile | 2 ++ webhook_server_container/libs/github_api.py | 5 +---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index b6d5d888..c024091d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,6 +31,7 @@ RUN dnf -y install dnf-plugins-core \ containerd.io \ docker-buildx-plugin \ docker-compose-plugin \ + slirp4netns \ && dnf clean all \ && rm -rf /var/cache /var/log/dnf* /var/log/yum.* @@ -45,6 +46,7 @@ RUN mkdir -p $BIN_DIR \ COPY entrypoint.sh pyproject.toml uv.lock README.md $APP_DIR/ COPY webhook_server_container $APP_DIR/webhook_server_container/ +RUN usermod --add-subuids 100000-165535 --add-subgids 100000-165535 $USERNAME RUN chown -R $USERNAME:$USERNAME $HOME_DIR USER $USERNAME WORKDIR $HOME_DIR diff --git a/webhook_server_container/libs/github_api.py b/webhook_server_container/libs/github_api.py index 4bacb881..9ceac659 100644 --- a/webhook_server_container/libs/github_api.py +++ b/webhook_server_container/libs/github_api.py @@ -1625,10 +1625,7 @@ def _run_in_container( checkout: str = "", tag_name: str = "", ) -> Tuple[int, str, str]: - podman_base_cmd: str = ( - "podman run --network=host --privileged -v /tmp/containers:/var/lib/containers/:Z " - f"--rm {env if env else ''} --entrypoint bash quay.io/myakove/github-webhook-server -c" - ) + podman_base_cmd: str = f"podman run --network=host --rm {env if env else ''} --entrypoint bash quay.io/myakove/github-webhook-server -c" # Clone the repository clone_base_cmd: str = ( From 070c7ba2b40d453fe4d89155f07a5b1e0b83b621 Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Sun, 13 Oct 2024 19:49:25 +0300 Subject: [PATCH 04/17] Use :noroot tag for now --- webhook_server_container/libs/github_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webhook_server_container/libs/github_api.py b/webhook_server_container/libs/github_api.py index 9ceac659..8b279308 100644 --- a/webhook_server_container/libs/github_api.py +++ b/webhook_server_container/libs/github_api.py @@ -1625,7 +1625,7 @@ def _run_in_container( checkout: str = "", tag_name: str = "", ) -> Tuple[int, str, str]: - podman_base_cmd: str = f"podman run --network=host --rm {env if env else ''} --entrypoint bash quay.io/myakove/github-webhook-server -c" + podman_base_cmd: str = f"podman run --network=host --rm {env if env else ''} --entrypoint bash quay.io/myakove/github-webhook-server:noroot -c" # Clone the repository clone_base_cmd: str = ( From c86f0ad43ffa61157d78ee49370c9b1fa25978e3 Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Sun, 13 Oct 2024 20:38:35 +0300 Subject: [PATCH 05/17] Use :noroot tag for now --- Dockerfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index c024091d..3a9a9185 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,7 +38,7 @@ RUN dnf -y install dnf-plugins-core \ RUN ln -s /usr/bin/python3 /usr/bin/python RUN mkdir -p $BIN_DIR \ - RUN mkdir -p $APP_DIR \ + && mkdir -p $APP_DIR \ && mkdir -p $DATA_DIR \ && mkdir -p $DATA_DIR/logs \ && mkdir -p /tmp/containers @@ -46,8 +46,9 @@ RUN mkdir -p $BIN_DIR \ COPY entrypoint.sh pyproject.toml uv.lock README.md $APP_DIR/ COPY webhook_server_container $APP_DIR/webhook_server_container/ -RUN usermod --add-subuids 100000-165535 --add-subgids 100000-165535 $USERNAME -RUN chown -R $USERNAME:$USERNAME $HOME_DIR +RUN usermod --add-subuids 100000-165535 --add-subgids 100000-165535 $USERNAME \ + && chown -R $USERNAME:$USERNAME $HOME_DIR + USER $USERNAME WORKDIR $HOME_DIR From 671812fcaa1c80630085544aaf710d56ffe2760e Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Mon, 14 Oct 2024 11:32:01 +0300 Subject: [PATCH 06/17] tox: dont use run_in_conatiner --- webhook_server_container/libs/github_api.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/webhook_server_container/libs/github_api.py b/webhook_server_container/libs/github_api.py index 8b279308..48878cf5 100644 --- a/webhook_server_container/libs/github_api.py +++ b/webhook_server_container/libs/github_api.py @@ -1,4 +1,5 @@ from __future__ import annotations +from uuid import uuid4 import contextlib import json import logging @@ -145,7 +146,7 @@ def __init__(self, hook_data: Dict[Any, Any], headers: Headers, logger: logging. return self.add_api_users_to_auto_verified_and_merged_users() - self.clone_repository_path: str = os.path.join("/", self.repository.name) + self.clone_repository_path: str = os.path.join("/", f"{self.clone_repository_path}-{uuid4()}") self.supported_user_labels_str: str = "".join([f" * {label}\n" for label in USER_LABELS_DICT.keys()]) self.welcome_msg: str = f""" @@ -1037,7 +1038,7 @@ def _run_tox(self) -> None: self.logger.debug(f"{self.log_prefix} Check run is in progress, not running {TOX_STR}.") return - cmd = f"{self.tox_python_version} -m {TOX_STR}" + cmd = f"{self.tox_python_version} -m {TOX_STR} --workdir {self.clone_repository_path} --root {self.clone_repository_path} -c {self.clone_repository_path}" _tox_tests = self.tox.get(self.pull_request_branch, "") if _tox_tests != "all": tests = _tox_tests.replace(" ", "") @@ -1625,7 +1626,7 @@ def _run_in_container( checkout: str = "", tag_name: str = "", ) -> Tuple[int, str, str]: - podman_base_cmd: str = f"podman run --network=host --rm {env if env else ''} --entrypoint bash quay.io/myakove/github-webhook-server:noroot -c" + # podman_base_cmd: str = f"podman run --network=host --rm {env if env else ''} --entrypoint bash quay.io/myakove/github-webhook-server:noroot -c" # Clone the repository clone_base_cmd: str = ( @@ -1661,8 +1662,8 @@ def _run_in_container( clone_base_cmd += f" && git checkout origin/pr/{pull_request.number}" # final podman command - podman_base_cmd += f" '{clone_base_cmd} && {command}'" - return run_command(command=podman_base_cmd, log_prefix=self.log_prefix) + # podman_base_cmd += f" '{clone_base_cmd} && {command}'" + return run_command(command=command, log_prefix=self.log_prefix) @staticmethod def get_check_run_text(err: str, out: str) -> str: From e286df3360c04317350f50674013891ffae5d993 Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Mon, 14 Oct 2024 11:37:02 +0300 Subject: [PATCH 07/17] tox: dont use run_in_conatiner --- webhook_server_container/libs/github_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webhook_server_container/libs/github_api.py b/webhook_server_container/libs/github_api.py index 48878cf5..aaf60874 100644 --- a/webhook_server_container/libs/github_api.py +++ b/webhook_server_container/libs/github_api.py @@ -146,7 +146,7 @@ def __init__(self, hook_data: Dict[Any, Any], headers: Headers, logger: logging. return self.add_api_users_to_auto_verified_and_merged_users() - self.clone_repository_path: str = os.path.join("/", f"{self.clone_repository_path}-{uuid4()}") + self.clone_repository_path: str = os.path.join("/", f"{self.repository.name}-{uuid4()}") self.supported_user_labels_str: str = "".join([f" * {label}\n" for label in USER_LABELS_DICT.keys()]) self.welcome_msg: str = f""" From 362e56f13c57f1f342bf2e70d16951e06282b78e Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Mon, 14 Oct 2024 11:54:19 +0300 Subject: [PATCH 08/17] tox: dont use run_in_conatiner --- webhook_server_container/libs/github_api.py | 40 ++++++++++++--------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/webhook_server_container/libs/github_api.py b/webhook_server_container/libs/github_api.py index aaf60874..2a29aacd 100644 --- a/webhook_server_container/libs/github_api.py +++ b/webhook_server_container/libs/github_api.py @@ -1,5 +1,6 @@ from __future__ import annotations from uuid import uuid4 +from github.Repository import Repository import contextlib import json import logging @@ -101,7 +102,7 @@ def __init__(self, hook_data: Dict[Any, Any], headers: Headers, logger: logging. self.headers = headers self.repository_name: str = hook_data["repository"]["name"] self.parent_committer: str = "" - self.container_repo_dir: str = "/tmp/repository" + self.clone_repo_dir: str = os.path.join("/", f"{self.repository.name}-{uuid4()}") self.jira_track_pr: bool = False self.issue_title: str = "" self.all_required_status_checks: List[str] = [] @@ -131,7 +132,9 @@ def __init__(self, hook_data: Dict[Any, Any], headers: Headers, logger: logging. ) if self.github_api and self.token: - self.repository = get_github_repo_api(github_api=self.github_api, repository=self.repository_full_name) + self.repository: Repository = get_github_repo_api( + github_api=self.github_api, repository=self.repository_full_name + ) else: self.logger.error(f"{self.log_prefix} Failed to get GitHub API and token.") @@ -146,7 +149,6 @@ def __init__(self, hook_data: Dict[Any, Any], headers: Headers, logger: logging. return self.add_api_users_to_auto_verified_and_merged_users() - self.clone_repository_path: str = os.path.join("/", f"{self.repository.name}-{uuid4()}") self.supported_user_labels_str: str = "".join([f" * {label}\n" for label in USER_LABELS_DICT.keys()]) self.welcome_msg: str = f""" @@ -396,7 +398,7 @@ def _repo_data_from_config(self) -> None: f"Project: {self.jira_project}, Token: {self.jira_token}" ) - self.auto_verified_and_merged_users = get_value_from_dicts( + self.auto_verified_and_merged_users: List[str] = get_value_from_dicts( primary_dict=repo_data, secondary_dict=config_data, key="auto-verified-and-merged-users", @@ -1038,7 +1040,7 @@ def _run_tox(self) -> None: self.logger.debug(f"{self.log_prefix} Check run is in progress, not running {TOX_STR}.") return - cmd = f"{self.tox_python_version} -m {TOX_STR} --workdir {self.clone_repository_path} --root {self.clone_repository_path} -c {self.clone_repository_path}" + cmd = f"{self.tox_python_version} -m {TOX_STR} --workdir {self.clone_repo_dir} --root {self.clone_repo_dir} -c {self.clone_repo_dir}" _tox_tests = self.tox.get(self.pull_request_branch, "") if _tox_tests != "all": tests = _tox_tests.replace(" ", "") @@ -1436,7 +1438,9 @@ def _run_build_container( _container_repository_and_tag = self._container_repository_and_tag(is_merged=is_merged, tag=tag) no_cache: str = " --no-cache" if is_merged else "" - build_cmd: str = f"--network=host {no_cache} -f {self.container_repo_dir}/{self.dockerfile} . -t {_container_repository_and_tag}" + build_cmd: str = ( + f"--network=host {no_cache} -f {self.clone_repo_dir}/{self.dockerfile} . -t {_container_repository_and_tag}" + ) if self.container_build_args: build_args: str = [f"--build-arg {b_arg}" for b_arg in self.container_build_args][0] @@ -1628,28 +1632,30 @@ def _run_in_container( ) -> Tuple[int, str, str]: # podman_base_cmd: str = f"podman run --network=host --rm {env if env else ''} --entrypoint bash quay.io/myakove/github-webhook-server:noroot -c" + git_cmd = f"git --work-tree={self.clone_repo_dir} --git-dir={self.clone_repo_dir}/.git" # Clone the repository clone_base_cmd: str = ( f"git clone {self.repository.clone_url.replace('https://', f'https://{self.token}@')} " - f"{self.container_repo_dir}" + f"{self.clone_repo_dir}" + ) + clone_base_cmd += f" && {git_cmd} config user.name '{self.repository.owner.login}'" + clone_base_cmd += f" && {git_cmd} config user.email '{self.repository.owner.email}'" + clone_base_cmd += ( + f" && {git_cmd} config --local --add remote.origin.fetch +refs/pull/*/head:refs/remotes/origin/pr/*" ) - clone_base_cmd += f" && cd {self.container_repo_dir}" - clone_base_cmd += f" && git config user.name '{self.repository.owner.login}'" - clone_base_cmd += f" && git config user.email '{self.repository.owner.email}'" - clone_base_cmd += " && git config --local --add remote.origin.fetch +refs/pull/*/head:refs/remotes/origin/pr/*" - clone_base_cmd += " && git remote update >/dev/null 2>&1" + clone_base_cmd += f" && {git_cmd} remote update >/dev/null 2>&1" # Checkout to requested branch/tag if checkout: - clone_base_cmd += f" && git checkout {checkout}" + clone_base_cmd += f" && {git_cmd} checkout {checkout}" # Checkout the branch if pull request is merged or for release else: if is_merged: - clone_base_cmd += f" && git checkout {self.pull_request_branch}" + clone_base_cmd += f" && {git_cmd} checkout {self.pull_request_branch}" elif tag_name: - clone_base_cmd += f" && git checkout {tag_name}" + clone_base_cmd += f" && {git_cmd} checkout {tag_name}" # Checkout the pull request else: @@ -1659,11 +1665,11 @@ def _run_in_container( self.logger.error(f"{self.log_prefix} [func:_run_in_container] No pull request found") return False, "", "" - clone_base_cmd += f" && git checkout origin/pr/{pull_request.number}" + clone_base_cmd += f" && {git_cmd} checkout origin/pr/{pull_request.number}" # final podman command # podman_base_cmd += f" '{clone_base_cmd} && {command}'" - return run_command(command=command, log_prefix=self.log_prefix) + return run_command(command=f"{clone_base_cmd} && {command}", log_prefix=self.log_prefix) @staticmethod def get_check_run_text(err: str, out: str) -> str: From e7dbef82c727b5f63b2ac7612d47d07af269bc34 Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Mon, 14 Oct 2024 11:59:11 +0300 Subject: [PATCH 09/17] tox: dont use run_in_conatiner --- webhook_server_container/libs/github_api.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/webhook_server_container/libs/github_api.py b/webhook_server_container/libs/github_api.py index 2a29aacd..7fe984d8 100644 --- a/webhook_server_container/libs/github_api.py +++ b/webhook_server_container/libs/github_api.py @@ -1,6 +1,5 @@ from __future__ import annotations from uuid import uuid4 -from github.Repository import Repository import contextlib import json import logging @@ -102,7 +101,6 @@ def __init__(self, hook_data: Dict[Any, Any], headers: Headers, logger: logging. self.headers = headers self.repository_name: str = hook_data["repository"]["name"] self.parent_committer: str = "" - self.clone_repo_dir: str = os.path.join("/", f"{self.repository.name}-{uuid4()}") self.jira_track_pr: bool = False self.issue_title: str = "" self.all_required_status_checks: List[str] = [] @@ -132,9 +130,7 @@ def __init__(self, hook_data: Dict[Any, Any], headers: Headers, logger: logging. ) if self.github_api and self.token: - self.repository: Repository = get_github_repo_api( - github_api=self.github_api, repository=self.repository_full_name - ) + self.repository = get_github_repo_api(github_api=self.github_api, repository=self.repository_full_name) else: self.logger.error(f"{self.log_prefix} Failed to get GitHub API and token.") @@ -148,6 +144,7 @@ def __init__(self, hook_data: Dict[Any, Any], headers: Headers, logger: logging. self.logger.error(f"{self.log_prefix} Failed to get repository.") return + self.clone_repo_dir: str = os.path.join("/", f"{self.repository.name}-{uuid4()}") self.add_api_users_to_auto_verified_and_merged_users() self.supported_user_labels_str: str = "".join([f" * {label}\n" for label in USER_LABELS_DICT.keys()]) From 62dba6e9b4a281bbe5daec3c62ddcab1639fab7e Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Mon, 14 Oct 2024 12:05:14 +0300 Subject: [PATCH 10/17] tox: dont use run_in_conatiner --- webhook_server_container/libs/github_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webhook_server_container/libs/github_api.py b/webhook_server_container/libs/github_api.py index 7fe984d8..29658381 100644 --- a/webhook_server_container/libs/github_api.py +++ b/webhook_server_container/libs/github_api.py @@ -144,7 +144,7 @@ def __init__(self, hook_data: Dict[Any, Any], headers: Headers, logger: logging. self.logger.error(f"{self.log_prefix} Failed to get repository.") return - self.clone_repo_dir: str = os.path.join("/", f"{self.repository.name}-{uuid4()}") + self.clone_repo_dir: str = os.path.join("/tmp", f"{self.repository.name}-{uuid4()}") self.add_api_users_to_auto_verified_and_merged_users() self.supported_user_labels_str: str = "".join([f" * {label}\n" for label in USER_LABELS_DICT.keys()]) From 6518644b52cf209398b7a93e5beff4b885c242b9 Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Mon, 14 Oct 2024 14:36:44 +0300 Subject: [PATCH 11/17] tox: dont use run_in_conatiner --- webhook_server_container/libs/github_api.py | 175 +++++++++++--------- 1 file changed, 101 insertions(+), 74 deletions(-) diff --git a/webhook_server_container/libs/github_api.py b/webhook_server_container/libs/github_api.py index 29658381..97f72209 100644 --- a/webhook_server_container/libs/github_api.py +++ b/webhook_server_container/libs/github_api.py @@ -1,6 +1,7 @@ from __future__ import annotations from uuid import uuid4 import contextlib +import shutil import json import logging import os @@ -144,7 +145,7 @@ def __init__(self, hook_data: Dict[Any, Any], headers: Headers, logger: logging. self.logger.error(f"{self.log_prefix} Failed to get repository.") return - self.clone_repo_dir: str = os.path.join("/tmp", f"{self.repository.name}-{uuid4()}") + self.clone_repo_dir: str = os.path.join("/tmp", f"{self.repository.name}") self.add_api_users_to_auto_verified_and_merged_users() self.supported_user_labels_str: str = "".join([f" * {label}\n" for label in USER_LABELS_DICT.keys()]) @@ -514,18 +515,20 @@ def is_branch_exists(self, branch: str) -> Branch: return self.repository.get_branch(branch) def upload_to_pypi(self, tag_name: str) -> None: + clone_repo_dir = f"{self.clone_repo_dir}-{uuid4()}" + uv_cmd_dir = f"--directory {clone_repo_dir}" out: str = "" - token: str = self.pypi["token"] - env: str = f"-e TWINE_USERNAME=__token__ -e TWINE_PASSWORD={token} " self.logger.info(f"{self.log_prefix} Start uploading to pypi") - _dist_dir: str = "/tmp/dist" + _dist_dir: str = f"{clone_repo_dir}/pypi-dist" cmd: str = ( - f" python3 -m build --sdist --outdir {_dist_dir} ." - f" && twine check {_dist_dir}/$(echo *.tar.gz)" - f" && twine upload {_dist_dir}/$(echo *.tar.gz) --skip-existing" + f"uv {uv_cmd_dir} build --sdist --out-dir {_dist_dir}" + f" && uvx {uv_cmd_dir} twine check {_dist_dir}/$(echo *.tar.gz)" + f" && uvx {uv_cmd_dir} twine upload --username __token__ --password {self.pypi["token"]} {_dist_dir}/$(echo *.tar.gz) --skip-existing" ) try: - rc, out, err = self._run_in_container(command=cmd, env=env, checkout=tag_name) + rc, out, err = self._run_command_in_cloned_repo( + command=cmd, checkout=tag_name, clone_repo_dir=clone_repo_dir + ) if rc: self.logger.info(f"{self.log_prefix} Publish to pypi finished") if self.slack_webhook_url: @@ -1037,14 +1040,15 @@ def _run_tox(self) -> None: self.logger.debug(f"{self.log_prefix} Check run is in progress, not running {TOX_STR}.") return - cmd = f"{self.tox_python_version} -m {TOX_STR} --workdir {self.clone_repo_dir} --root {self.clone_repo_dir} -c {self.clone_repo_dir}" + clone_repo_dir = f"{self.clone_repo_dir}-{uuid4()}" + cmd = f"uvx --python={self.tox_python_version} {TOX_STR} --workdir {clone_repo_dir} --root {clone_repo_dir} -c {clone_repo_dir}" _tox_tests = self.tox.get(self.pull_request_branch, "") if _tox_tests != "all": tests = _tox_tests.replace(" ", "") cmd += f" -e {tests}" self.set_run_tox_check_in_progress() - rc, out, err = self._run_in_container(command=cmd) + rc, out, err = self._run_command_in_cloned_repo(command=cmd, clone_repo_dir=clone_repo_dir) output: Dict[str, Any] = { "title": "Tox", @@ -1064,9 +1068,10 @@ def _run_pre_commit(self) -> None: self.logger.debug(f"{self.log_prefix} Check run is in progress, not running {PRE_COMMIT_STR}.") return - cmd = f"{PRE_COMMIT_STR} run --all-files" + clone_repo_dir = f"{self.clone_repo_dir}-{uuid4()}" + cmd = f" uvx --directory {clone_repo_dir} {PRE_COMMIT_STR} run --all-files" self.set_run_pre_commit_check_in_progress() - rc, out, err = self._run_in_container(command=cmd) + rc, out, err = self._run_command_in_cloned_repo(command=cmd, clone_repo_dir=clone_repo_dir) output: Dict[str, Any] = { "title": "Pre-Commit", @@ -1174,27 +1179,30 @@ def cherry_pick(self, target_branch: str, reviewed_user: str = "") -> None: err_msg = f"cherry-pick failed: {target_branch} does not exists" self.logger.error(err_msg) self.pull_request.create_issue_comment(err_msg) + else: self.set_cherry_pick_in_progress() commit_hash = self.pull_request.merge_commit_sha commit_msg_striped = self.pull_request.title.replace("'", "") pull_request_url = self.pull_request.html_url - env = f"-e GITHUB_TOKEN={self.token}" + clone_repo_dir = f"{self.clone_repo_dir}-{uuid4()}" + git_cmd = f"GITHUB_TOKEN={self.token} git --work-tree={clone_repo_dir} --git-dir={clone_repo_dir}/.git" + hub_cmd = f"GITHUB_TOKEN={self.token} hub --work-tree={clone_repo_dir} --git-dir={clone_repo_dir}/.git" cmd = ( - f" git checkout {target_branch}" - f" && git pull origin {target_branch}" - f" && git checkout -b {new_branch_name} origin/{target_branch}" - f" && git cherry-pick {commit_hash}" - f" && git push origin {new_branch_name}" - f" && hub pull-request " - f"-b {target_branch} " - f"-h {new_branch_name} " - f"-l {CHERRY_PICKED_LABEL_PREFIX} " - f'-m "{CHERRY_PICKED_LABEL_PREFIX}: [{target_branch}] {commit_msg_striped}" ' - f'-m "cherry-pick {pull_request_url} into {target_branch}" ' - f'-m "requested-by {requested_by}"' + f" {git_cmd} checkout {target_branch}" + f" && {git_cmd} pull origin {target_branch}" + f" && {git_cmd} checkout -b {new_branch_name} origin/{target_branch}" + f" && {git_cmd} cherry-pick {commit_hash}" + f" && {git_cmd} push origin {new_branch_name}" + f" && {hub_cmd} pull-request" + f" -b {target_branch}" + f" -h {new_branch_name}" + f" -l {CHERRY_PICKED_LABEL_PREFIX}" + f' -m "{CHERRY_PICKED_LABEL_PREFIX}: [{target_branch}] {commit_msg_striped}"' + f' -m "cherry-pick {pull_request_url} into {target_branch}"' + f' -m "requested-by {requested_by}"' ) - rc, out, err = self._run_in_container(command=cmd, env=env) + rc, out, err = self._run_command_in_cloned_repo(command=cmd, clone_repo_dir=clone_repo_dir) output = { "title": "Cherry-pick details", @@ -1424,6 +1432,7 @@ def _run_build_container( if not self.build_and_push_container: return + clone_repo_dir = f"{self.clone_repo_dir}-{uuid4()}" pull_request = hasattr(self, "pull_request") if pull_request and set_check: @@ -1436,7 +1445,7 @@ def _run_build_container( _container_repository_and_tag = self._container_repository_and_tag(is_merged=is_merged, tag=tag) no_cache: str = " --no-cache" if is_merged else "" build_cmd: str = ( - f"--network=host {no_cache} -f {self.clone_repo_dir}/{self.dockerfile} . -t {_container_repository_and_tag}" + f"--network=host {no_cache} -f {clone_repo_dir}/{self.dockerfile} . -t {_container_repository_and_tag}" ) if self.container_build_args: @@ -1449,23 +1458,34 @@ def _run_build_container( if command_args: build_cmd = f"{command_args} {build_cmd}" - if push: - repository_creds: str = f"{self.container_repository_username}:{self.container_repository_password}" - build_cmd += f" && podman push --creds {repository_creds} {_container_repository_and_tag}" podman_build_cmd: str = f"podman build {build_cmd}" + build_rc, build_out, build_err = self._run_command_in_cloned_repo( + command=podman_build_cmd, + is_merged=is_merged, + tag_name=tag, + clone_repo_dir=clone_repo_dir, + ) - rc, out, err = self._run_in_container(command=podman_build_cmd, is_merged=is_merged, tag_name=tag) output: Dict[str, str] = { "title": "Build container", "summary": "", - "text": self.get_check_run_text(err=err, out=out), + "text": self.get_check_run_text(err=build_err, out=build_out), } - if rc: + if build_rc: self.logger.info(f"{self.log_prefix} Done building {_container_repository_and_tag}") if pull_request and set_check: return self.set_container_build_success(output=output) + else: + self.logger.error(f"{self.log_prefix} Failed to build {_container_repository_and_tag}") + if self.pull_request and set_check: + return self.set_container_build_failure(output=output) - if push: + if push: + push_rc = self._run_command_in_cloned_repo( + command=f"podman push --creds {self.container_repository_username}:{self.container_repository_password} {_container_repository_and_tag}", + clone_repo_dir=clone_repo_dir, + ) + if push_rc: push_msg: str = f"New container for {_container_repository_and_tag} published" if pull_request: self.pull_request.create_issue_comment(push_msg) @@ -1479,8 +1499,7 @@ def _run_build_container( self.send_slack_message(message=message, webhook_url=self.slack_webhook_url) self.logger.info(f"{self.log_prefix} Done push {_container_repository_and_tag}") - else: - if push: + else: err_msg: str = f"Failed to build and push {_container_repository_and_tag}" if self.pull_request: self.pull_request.create_issue_comment(err_msg) @@ -1493,9 +1512,6 @@ def _run_build_container( """ self.send_slack_message(message=message, webhook_url=self.slack_webhook_url) - if self.pull_request and set_check: - return self.set_container_build_failure(output=output) - def _run_install_python_module(self) -> None: if not self.pypi: return @@ -1504,10 +1520,13 @@ def _run_install_python_module(self) -> None: self.logger.info(f"{self.log_prefix} Check run is in progress, not running {PYTHON_MODULE_INSTALL_STR}.") return + clone_repo_dir = f"{self.clone_repo_dir}-{uuid4()}" self.logger.info(f"{self.log_prefix} Installing python module") - f"{PYTHON_MODULE_INSTALL_STR}-{shortuuid.uuid()}" self.set_python_module_install_in_progress() - rc, out, err = self._run_in_container(command="pip install .") + rc, out, err = self._run_command_in_cloned_repo( + command=f"uvx pip wheel --no-cache-dir -w {clone_repo_dir}/dist {clone_repo_dir}", + clone_repo_dir=clone_repo_dir, + ) output: Dict[str, str] = { "title": "Python module installation", "summary": "", @@ -1619,54 +1638,62 @@ def set_check_run_status( kwargs["conclusion"] = FAILURE_STR self.repository_by_github_app.create_check_run(**kwargs) - def _run_in_container( + def _run_command_in_cloned_repo( self, command: str, - env: str = "", + clone_repo_dir: str, is_merged: bool = False, checkout: str = "", tag_name: str = "", ) -> Tuple[int, str, str]: - # podman_base_cmd: str = f"podman run --network=host --rm {env if env else ''} --entrypoint bash quay.io/myakove/github-webhook-server:noroot -c" + git_cmd = f"git --work-tree={clone_repo_dir} --git-dir={clone_repo_dir}/.git" - git_cmd = f"git --work-tree={self.clone_repo_dir} --git-dir={self.clone_repo_dir}/.git" # Clone the repository - clone_base_cmd: str = ( - f"git clone {self.repository.clone_url.replace('https://', f'https://{self.token}@')} " - f"{self.clone_repo_dir}" - ) - clone_base_cmd += f" && {git_cmd} config user.name '{self.repository.owner.login}'" - clone_base_cmd += f" && {git_cmd} config user.email '{self.repository.owner.email}'" - clone_base_cmd += ( - f" && {git_cmd} config --local --add remote.origin.fetch +refs/pull/*/head:refs/remotes/origin/pr/*" + run_command( + command=f"git clone {self.repository.clone_url.replace('https://', f'https://{self.token}@')} " + f"{clone_repo_dir}", + log_prefix=self.log_prefix, ) - clone_base_cmd += f" && {git_cmd} remote update >/dev/null 2>&1" + try: + run_command( + command=f"{git_cmd} config user.name '{self.repository.owner.login}'", log_prefix=self.log_prefix + ) + run_command(f"{git_cmd} config user.email '{self.repository.owner.email}'", log_prefix=self.log_prefix) + run_command( + command=f"{git_cmd} config --local --add remote.origin.fetch +refs/pull/*/head:refs/remotes/origin/pr/*", + log_prefix=self.log_prefix, + ) + run_command(command=f"{git_cmd} remote update", log_prefix=self.log_prefix) - # Checkout to requested branch/tag - if checkout: - clone_base_cmd += f" && {git_cmd} checkout {checkout}" + # Checkout to requested branch/tag + if checkout: + run_command(f"{git_cmd} checkout {checkout}", log_prefix=self.log_prefix) - # Checkout the branch if pull request is merged or for release - else: - if is_merged: - clone_base_cmd += f" && {git_cmd} checkout {self.pull_request_branch}" + # Checkout the branch if pull request is merged or for release + else: + if is_merged: + run_command(command=f"{git_cmd} checkout {self.pull_request_branch}", log_prefix=self.log_prefix) - elif tag_name: - clone_base_cmd += f" && {git_cmd} checkout {tag_name}" + elif tag_name: + run_command(command=f"{git_cmd} checkout {tag_name}", log_prefix=self.log_prefix) - # Checkout the pull request - else: - try: - pull_request = self._get_pull_request() - except NoPullRequestError: - self.logger.error(f"{self.log_prefix} [func:_run_in_container] No pull request found") - return False, "", "" + # Checkout the pull request + else: + try: + pull_request = self._get_pull_request() + except NoPullRequestError: + self.logger.error(f"{self.log_prefix} [func:_run_in_container] No pull request found") + return False, "", "" + + run_command( + command=f"{git_cmd} checkout origin/pr/{pull_request.number}", log_prefix=self.log_prefix + ) - clone_base_cmd += f" && {git_cmd} checkout origin/pr/{pull_request.number}" + return run_command(command=command, log_prefix=self.log_prefix) - # final podman command - # podman_base_cmd += f" '{clone_base_cmd} && {command}'" - return run_command(command=f"{clone_base_cmd} && {command}", log_prefix=self.log_prefix) + finally: + self.logger.debug(f"{self.log_prefix} Deleting {clone_repo_dir}") + shutil.rmtree(clone_repo_dir) @staticmethod def get_check_run_text(err: str, out: str) -> str: From 808a0b154a79d5515aceb55fba50f00d396b6d53 Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Mon, 14 Oct 2024 15:48:44 +0300 Subject: [PATCH 12/17] testfile --- webhook_server_container/libs/github_api.py | 283 ++++++++++---------- 1 file changed, 148 insertions(+), 135 deletions(-) diff --git a/webhook_server_container/libs/github_api.py b/webhook_server_container/libs/github_api.py index 97f72209..3664266d 100644 --- a/webhook_server_container/libs/github_api.py +++ b/webhook_server_container/libs/github_api.py @@ -10,7 +10,7 @@ import time from concurrent.futures import Future, ThreadPoolExecutor, as_completed from pathlib import Path -from typing import Any, Dict, List, Optional, Set, Tuple +from typing import Any, Dict, Generator, List, Optional, Set from stringcolor import cs from github.Branch import Branch @@ -520,36 +520,36 @@ def upload_to_pypi(self, tag_name: str) -> None: out: str = "" self.logger.info(f"{self.log_prefix} Start uploading to pypi") _dist_dir: str = f"{clone_repo_dir}/pypi-dist" - cmd: str = ( - f"uv {uv_cmd_dir} build --sdist --out-dir {_dist_dir}" - f" && uvx {uv_cmd_dir} twine check {_dist_dir}/$(echo *.tar.gz)" - f" && uvx {uv_cmd_dir} twine upload --username __token__ --password {self.pypi["token"]} {_dist_dir}/$(echo *.tar.gz) --skip-existing" - ) - try: - rc, out, err = self._run_command_in_cloned_repo( - command=cmd, checkout=tag_name, clone_repo_dir=clone_repo_dir - ) - if rc: - self.logger.info(f"{self.log_prefix} Publish to pypi finished") - if self.slack_webhook_url: - message: str = f""" -``` -{self.repository_name} Version {tag_name} published to PYPI. -``` -""" - self.send_slack_message(message=message, webhook_url=self.slack_webhook_url) - except Exception as exp: - err = f"Publish to pypi failed: {exp}" - self.logger.error(f"{self.log_prefix} {err}") - self.repository.create_issue( - title=err, - assignee=self.approvers[0] if self.approvers else "", - body=f""" + commands: List[str] = [ + f"uv {uv_cmd_dir} build --sdist --out-dir {_dist_dir}", + f"uvx {uv_cmd_dir} twine check {_dist_dir}/$(echo *.tar.gz)", + f"uvx {uv_cmd_dir} twine upload --username __token__ --password {self.pypi["token"]} {_dist_dir}/$(echo *.tar.gz) --skip-existing", + ] + with self._prepare_cloned_repo_dir(checkout=tag_name, clone_repo_dir=clone_repo_dir): + for cmd in commands: + rc, out, err = run_command(command=cmd, log_prefix=self.log_prefix) + if not rc: + err = "Publish to pypi failed" + self.logger.error(f"{self.log_prefix} {err}") + self.repository.create_issue( + title=err, + assignee=self.approvers[0] if self.approvers else "", + body=f""" stdout: `{out}` stderr: `{err}` """, - ) + ) + return + + self.logger.info(f"{self.log_prefix} Publish to pypi finished") + if self.slack_webhook_url: + message: str = f""" +``` +{self.repository_name} Version {tag_name} published to PYPI. +``` +""" + self.send_slack_message(message=message, webhook_url=self.slack_webhook_url) def get_owners_content(self) -> Dict[str, Any]: try: @@ -1048,17 +1048,18 @@ def _run_tox(self) -> None: cmd += f" -e {tests}" self.set_run_tox_check_in_progress() - rc, out, err = self._run_command_in_cloned_repo(command=cmd, clone_repo_dir=clone_repo_dir) + with self._prepare_cloned_repo_dir(clone_repo_dir=clone_repo_dir): + rc, out, err = run_command(command=cmd, log_prefix=self.log_prefix) - output: Dict[str, Any] = { - "title": "Tox", - "summary": "", - "text": self.get_check_run_text(err=err, out=out), - } - if rc: - return self.set_run_tox_check_success(output=output) - else: - return self.set_run_tox_check_failure(output=output) + output: Dict[str, Any] = { + "title": "Tox", + "summary": "", + "text": self.get_check_run_text(err=err, out=out), + } + if rc: + return self.set_run_tox_check_success(output=output) + else: + return self.set_run_tox_check_failure(output=output) def _run_pre_commit(self) -> None: if not self.pre_commit: @@ -1071,17 +1072,18 @@ def _run_pre_commit(self) -> None: clone_repo_dir = f"{self.clone_repo_dir}-{uuid4()}" cmd = f" uvx --directory {clone_repo_dir} {PRE_COMMIT_STR} run --all-files" self.set_run_pre_commit_check_in_progress() - rc, out, err = self._run_command_in_cloned_repo(command=cmd, clone_repo_dir=clone_repo_dir) + with self._prepare_cloned_repo_dir(clone_repo_dir=clone_repo_dir): + rc, out, err = run_command(command=cmd, log_prefix=self.log_prefix) - output: Dict[str, Any] = { - "title": "Pre-Commit", - "summary": "", - "text": self.get_check_run_text(err=err, out=out), - } - if rc: - return self.set_run_pre_commit_check_success(output=output) - else: - return self.set_run_pre_commit_check_failure(output=output) + output: Dict[str, Any] = { + "title": "Pre-Commit", + "summary": "", + "text": self.get_check_run_text(err=err, out=out), + } + if rc: + return self.set_run_pre_commit_check_success(output=output) + else: + return self.set_run_pre_commit_check_failure(output=output) def user_commands(self, command: str, reviewed_user: str, issue_comment_id: int) -> None: available_commands: List[str] = [ @@ -1186,51 +1188,58 @@ def cherry_pick(self, target_branch: str, reviewed_user: str = "") -> None: commit_msg_striped = self.pull_request.title.replace("'", "") pull_request_url = self.pull_request.html_url clone_repo_dir = f"{self.clone_repo_dir}-{uuid4()}" - git_cmd = f"GITHUB_TOKEN={self.token} git --work-tree={clone_repo_dir} --git-dir={clone_repo_dir}/.git" + git_cmd = f"git --work-tree={clone_repo_dir} --git-dir={clone_repo_dir}/.git" hub_cmd = f"GITHUB_TOKEN={self.token} hub --work-tree={clone_repo_dir} --git-dir={clone_repo_dir}/.git" - cmd = ( - f" {git_cmd} checkout {target_branch}" - f" && {git_cmd} pull origin {target_branch}" - f" && {git_cmd} checkout -b {new_branch_name} origin/{target_branch}" - f" && {git_cmd} cherry-pick {commit_hash}" - f" && {git_cmd} push origin {new_branch_name}" - f" && {hub_cmd} pull-request" + commands: List[str] = [ + f"{git_cmd} checkout {target_branch}", + f"{git_cmd} pull origin {target_branch}", + f"{git_cmd} checkout -b {new_branch_name} origin/{target_branch}", + f"{git_cmd} cherry-pick {commit_hash}", + f"{git_cmd} push origin {new_branch_name}", + f"{hub_cmd} pull-request" f" -b {target_branch}" f" -h {new_branch_name}" f" -l {CHERRY_PICKED_LABEL_PREFIX}" f' -m "{CHERRY_PICKED_LABEL_PREFIX}: [{target_branch}] {commit_msg_striped}"' f' -m "cherry-pick {pull_request_url} into {target_branch}"' - f' -m "requested-by {requested_by}"' - ) - rc, out, err = self._run_command_in_cloned_repo(command=cmd, clone_repo_dir=clone_repo_dir) + f' -m "requested-by {requested_by}"', + ] + + rc, out, err = None, "", "" + with self._prepare_cloned_repo_dir(clone_repo_dir=clone_repo_dir): + for cmd in commands: + rc, out, err = run_command(command=cmd, log_prefix=self.log_prefix) + if not rc: + output = { + "title": "Cherry-pick details", + "summary": "", + "text": self.get_check_run_text(err=err, out=out), + } + self.set_cherry_pick_failure(output=output) + self.logger.error(f"{self.log_prefix} Cherry pick failed: {out} --- {err}") + local_branch_name = f"{self.pull_request.head.ref}-{target_branch}" + self.pull_request.create_issue_comment( + f"**Manual cherry-pick is needed**\nCherry pick failed for " + f"{commit_hash} to {target_branch}:\n" + f"To cherry-pick run:\n" + "```\n" + f"git remote update\n" + f"git checkout {target_branch}\n" + f"git pull origin {target_branch}\n" + f"git checkout -b {local_branch_name}\n" + f"git cherry-pick {commit_hash}\n" + f"git push origin {local_branch_name}\n" + "```" + ) + return output = { "title": "Cherry-pick details", "summary": "", "text": self.get_check_run_text(err=err, out=out), } - if rc: - self.set_cherry_pick_success(output=output) - self.pull_request.create_issue_comment( - f"Cherry-picked PR {self.pull_request.title} into {target_branch}" - ) - else: - self.set_cherry_pick_failure(output=output) - self.logger.error(f"{self.log_prefix} Cherry pick failed: {out} --- {err}") - local_branch_name = f"{self.pull_request.head.ref}-{target_branch}" - self.pull_request.create_issue_comment( - f"**Manual cherry-pick is needed**\nCherry pick failed for " - f"{commit_hash} to {target_branch}:\n" - f"To cherry-pick run:\n" - "```\n" - f"git remote update\n" - f"git checkout {target_branch}\n" - f"git pull origin {target_branch}\n" - f"git checkout -b {local_branch_name}\n" - f"git cherry-pick {commit_hash}\n" - f"git push origin {local_branch_name}\n" - "```" - ) + self.set_cherry_pick_success(output=output) + self.pull_request.create_issue_comment(f"Cherry-picked PR {self.pull_request.title} into {target_branch}") def label_all_opened_pull_requests_merge_state_after_merged(self) -> None: """ @@ -1459,58 +1468,58 @@ def _run_build_container( build_cmd = f"{command_args} {build_cmd}" podman_build_cmd: str = f"podman build {build_cmd}" - build_rc, build_out, build_err = self._run_command_in_cloned_repo( - command=podman_build_cmd, + with self._prepare_cloned_repo_dir( is_merged=is_merged, tag_name=tag, clone_repo_dir=clone_repo_dir, - ) - - output: Dict[str, str] = { - "title": "Build container", - "summary": "", - "text": self.get_check_run_text(err=build_err, out=build_out), - } - if build_rc: - self.logger.info(f"{self.log_prefix} Done building {_container_repository_and_tag}") - if pull_request and set_check: - return self.set_container_build_success(output=output) - else: - self.logger.error(f"{self.log_prefix} Failed to build {_container_repository_and_tag}") - if self.pull_request and set_check: - return self.set_container_build_failure(output=output) - - if push: - push_rc = self._run_command_in_cloned_repo( - command=f"podman push --creds {self.container_repository_username}:{self.container_repository_password} {_container_repository_and_tag}", - clone_repo_dir=clone_repo_dir, + ): + build_rc, build_out, build_err = run_command( + command=podman_build_cmd, + log_prefix=self.log_prefix, ) - if push_rc: - push_msg: str = f"New container for {_container_repository_and_tag} published" - if pull_request: - self.pull_request.create_issue_comment(push_msg) - - if self.slack_webhook_url: - message = f""" + output: Dict[str, str] = { + "title": "Build container", + "summary": "", + "text": self.get_check_run_text(err=build_err, out=build_out), + } + if build_rc: + self.logger.info(f"{self.log_prefix} Done building {_container_repository_and_tag}") + if pull_request and set_check: + return self.set_container_build_success(output=output) + else: + self.logger.error(f"{self.log_prefix} Failed to build {_container_repository_and_tag}") + if self.pull_request and set_check: + return self.set_container_build_failure(output=output) + + if push: + cmd = f"podman push --creds {self.container_repository_username}:{self.container_repository_password} {_container_repository_and_tag}" + push_rc, _, _ = run_command(command=cmd, log_prefix=self.log_prefix) + if push_rc: + push_msg: str = f"New container for {_container_repository_and_tag} published" + if pull_request: + self.pull_request.create_issue_comment(push_msg) + + if self.slack_webhook_url: + message = f""" ``` {self.repository_full_name} {push_msg}. ``` """ - self.send_slack_message(message=message, webhook_url=self.slack_webhook_url) + self.send_slack_message(message=message, webhook_url=self.slack_webhook_url) - self.logger.info(f"{self.log_prefix} Done push {_container_repository_and_tag}") - else: - err_msg: str = f"Failed to build and push {_container_repository_and_tag}" - if self.pull_request: - self.pull_request.create_issue_comment(err_msg) + self.logger.info(f"{self.log_prefix} Done push {_container_repository_and_tag}") + else: + err_msg: str = f"Failed to build and push {_container_repository_and_tag}" + if self.pull_request: + self.pull_request.create_issue_comment(err_msg) - if self.slack_webhook_url: - message = f""" + if self.slack_webhook_url: + message = f""" ``` {self.repository_full_name} {err_msg}. ``` - """ - self.send_slack_message(message=message, webhook_url=self.slack_webhook_url) + """ + self.send_slack_message(message=message, webhook_url=self.slack_webhook_url) def _run_install_python_module(self) -> None: if not self.pypi: @@ -1523,19 +1532,23 @@ def _run_install_python_module(self) -> None: clone_repo_dir = f"{self.clone_repo_dir}-{uuid4()}" self.logger.info(f"{self.log_prefix} Installing python module") self.set_python_module_install_in_progress() - rc, out, err = self._run_command_in_cloned_repo( - command=f"uvx pip wheel --no-cache-dir -w {clone_repo_dir}/dist {clone_repo_dir}", + with self._prepare_cloned_repo_dir( clone_repo_dir=clone_repo_dir, - ) - output: Dict[str, str] = { - "title": "Python module installation", - "summary": "", - "text": self.get_check_run_text(err=err, out=out), - } - if rc: - return self.set_python_module_install_success(output=output) + ): + rc, out, err = run_command( + command=f"uvx pip wheel --no-cache-dir -w {clone_repo_dir}/dist {clone_repo_dir}", + log_prefix=self.log_prefix, + ) + + output: Dict[str, str] = { + "title": "Python module installation", + "summary": "", + "text": self.get_check_run_text(err=err, out=out), + } + if rc: + return self.set_python_module_install_success(output=output) - return self.set_python_module_install_failure(output=output) + return self.set_python_module_install_failure(output=output) def send_slack_message(self, message: str, webhook_url: str) -> None: slack_data: Dict[str, str] = {"text": message} @@ -1638,14 +1651,14 @@ def set_check_run_status( kwargs["conclusion"] = FAILURE_STR self.repository_by_github_app.create_check_run(**kwargs) - def _run_command_in_cloned_repo( + @contextlib.contextmanager + def _prepare_cloned_repo_dir( self, - command: str, clone_repo_dir: str, is_merged: bool = False, checkout: str = "", tag_name: str = "", - ) -> Tuple[int, str, str]: + ) -> Generator[None, None, None]: git_cmd = f"git --work-tree={clone_repo_dir} --git-dir={clone_repo_dir}/.git" # Clone the repository @@ -1683,13 +1696,13 @@ def _run_command_in_cloned_repo( pull_request = self._get_pull_request() except NoPullRequestError: self.logger.error(f"{self.log_prefix} [func:_run_in_container] No pull request found") - return False, "", "" + return run_command( command=f"{git_cmd} checkout origin/pr/{pull_request.number}", log_prefix=self.log_prefix ) - return run_command(command=command, log_prefix=self.log_prefix) + yield finally: self.logger.debug(f"{self.log_prefix} Deleting {clone_repo_dir}") From dc1c4a2d06d4b23cff880f4fef713ae1f6d7cbdf Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Mon, 14 Oct 2024 16:23:08 +0300 Subject: [PATCH 13/17] fix pypi release --- webhook_server_container/libs/github_api.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/webhook_server_container/libs/github_api.py b/webhook_server_container/libs/github_api.py index 3664266d..0ccc4350 100644 --- a/webhook_server_container/libs/github_api.py +++ b/webhook_server_container/libs/github_api.py @@ -521,12 +521,15 @@ def upload_to_pypi(self, tag_name: str) -> None: self.logger.info(f"{self.log_prefix} Start uploading to pypi") _dist_dir: str = f"{clone_repo_dir}/pypi-dist" - commands: List[str] = [ - f"uv {uv_cmd_dir} build --sdist --out-dir {_dist_dir}", - f"uvx {uv_cmd_dir} twine check {_dist_dir}/$(echo *.tar.gz)", - f"uvx {uv_cmd_dir} twine upload --username __token__ --password {self.pypi["token"]} {_dist_dir}/$(echo *.tar.gz) --skip-existing", - ] with self._prepare_cloned_repo_dir(checkout=tag_name, clone_repo_dir=clone_repo_dir): + rc, tar_gz_file, err = run_command(command=f"ls {_dist_dir}", log_prefix=self.log_prefix) + tar_gz_file = tar_gz_file.strip() + + commands: List[str] = [ + f"uv {uv_cmd_dir} build --sdist --out-dir {_dist_dir}", + f"uvx {uv_cmd_dir} twine check {_dist_dir}/{tar_gz_file}", + f"uvx {uv_cmd_dir} twine upload --username __token__ --password {self.pypi["token"]} {_dist_dir}/{tar_gz_file} --skip-existing", + ] for cmd in commands: rc, out, err = run_command(command=cmd, log_prefix=self.log_prefix) if not rc: From 6e6fc0fb32bf048a8763df6f831c410144adafb8 Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Mon, 14 Oct 2024 16:38:46 +0300 Subject: [PATCH 14/17] fix pypi release --- webhook_server_container/libs/github_api.py | 34 +++++++++++++-------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/webhook_server_container/libs/github_api.py b/webhook_server_container/libs/github_api.py index 0ccc4350..16c669ad 100644 --- a/webhook_server_container/libs/github_api.py +++ b/webhook_server_container/libs/github_api.py @@ -515,6 +515,18 @@ def is_branch_exists(self, branch: str) -> Branch: return self.repository.get_branch(branch) def upload_to_pypi(self, tag_name: str) -> None: + def _error(_out: str, _err: str) -> None: + err: str = "Publish to pypi failed" + self.logger.error(f"{self.log_prefix} {err} - {_err}, {_out}") + self.repository.create_issue( + title=_err, + assignee=self.approvers[0] if self.approvers else "", + body=f""" +stdout: `{_out}` +stderr: `{_err}` +""", + ) + clone_repo_dir = f"{self.clone_repo_dir}-{uuid4()}" uv_cmd_dir = f"--directory {clone_repo_dir}" out: str = "" @@ -522,28 +534,26 @@ def upload_to_pypi(self, tag_name: str) -> None: _dist_dir: str = f"{clone_repo_dir}/pypi-dist" with self._prepare_cloned_repo_dir(checkout=tag_name, clone_repo_dir=clone_repo_dir): + rc, out, err = run_command( + command=f"uv {uv_cmd_dir} build --sdist --out-dir {_dist_dir}", log_prefix=self.log_prefix + ) + if not rc: + return _error(_out=out, _err=err) + rc, tar_gz_file, err = run_command(command=f"ls {_dist_dir}", log_prefix=self.log_prefix) + if not rc: + return _error(_out=out, _err=err) + tar_gz_file = tar_gz_file.strip() commands: List[str] = [ - f"uv {uv_cmd_dir} build --sdist --out-dir {_dist_dir}", f"uvx {uv_cmd_dir} twine check {_dist_dir}/{tar_gz_file}", f"uvx {uv_cmd_dir} twine upload --username __token__ --password {self.pypi["token"]} {_dist_dir}/{tar_gz_file} --skip-existing", ] for cmd in commands: rc, out, err = run_command(command=cmd, log_prefix=self.log_prefix) if not rc: - err = "Publish to pypi failed" - self.logger.error(f"{self.log_prefix} {err}") - self.repository.create_issue( - title=err, - assignee=self.approvers[0] if self.approvers else "", - body=f""" -stdout: `{out}` -stderr: `{err}` -""", - ) - return + return _error(_out=out, _err=err) self.logger.info(f"{self.log_prefix} Publish to pypi finished") if self.slack_webhook_url: From 5daaebd32494399e23433ff8b5a89b2dee89d667 Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Mon, 14 Oct 2024 17:38:54 +0300 Subject: [PATCH 15/17] Fix cherry-pick --- Dockerfile | 28 ++++++++++----------- webhook_server_container/libs/github_api.py | 8 +----- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3a9a9185..348de3c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,7 @@ RUN dnf -y install dnf-plugins-core \ && dnf clean all \ && rm -rf /var/cache /var/log/dnf* /var/log/yum.* -RUN ln -s /usr/bin/python3 /usr/bin/python +# RUN ln -s /usr/bin/python3 /usr/bin/python RUN mkdir -p $BIN_DIR \ && mkdir -p $APP_DIR \ @@ -64,19 +64,19 @@ RUN set -x \ && chmod +x $BIN_DIR/rosa \ && rm -rf $BIN_DIR/rosa-linux.tar.gz -RUN python -m pip install --no-cache-dir pip --upgrade \ - && python -m pip install --no-cache-dir poetry tox twine pre-commit - -RUN python3.8 -m ensurepip \ - && python3.9 -m ensurepip \ - && python3.10 -m ensurepip \ - && python3.11 -m ensurepip \ - && python3.12 -m ensurepip \ - && python3.8 -m pip install tox \ - && python3.9 -m pip install tox \ - && python3.10 -m pip install tox \ - && python3.11 -m pip install tox \ - && python3.12 -m pip install tox +# RUN python -m pip install --no-cache-dir pip --upgrade \ +# && python -m pip install --no-cache-dir poetry tox twine pre-commit +# +# RUN python3.8 -m ensurepip \ +# && python3.9 -m ensurepip \ +# && python3.10 -m ensurepip \ +# && python3.11 -m ensurepip \ +# && python3.12 -m ensurepip \ +# && python3.8 -m pip install tox \ +# && python3.9 -m pip install tox \ +# && python3.10 -m pip install tox \ +# && python3.11 -m pip install tox \ +# && python3.12 -m pip install tox WORKDIR $APP_DIR diff --git a/webhook_server_container/libs/github_api.py b/webhook_server_container/libs/github_api.py index 16c669ad..160d1cf9 100644 --- a/webhook_server_container/libs/github_api.py +++ b/webhook_server_container/libs/github_api.py @@ -1209,13 +1209,7 @@ def cherry_pick(self, target_branch: str, reviewed_user: str = "") -> None: f"{git_cmd} checkout -b {new_branch_name} origin/{target_branch}", f"{git_cmd} cherry-pick {commit_hash}", f"{git_cmd} push origin {new_branch_name}", - f"{hub_cmd} pull-request" - f" -b {target_branch}" - f" -h {new_branch_name}" - f" -l {CHERRY_PICKED_LABEL_PREFIX}" - f' -m "{CHERRY_PICKED_LABEL_PREFIX}: [{target_branch}] {commit_msg_striped}"' - f' -m "cherry-pick {pull_request_url} into {target_branch}"' - f' -m "requested-by {requested_by}"', + f"bash -c \"{hub_cmd} pull-request -b {target_branch} -h {new_branch_name} -l {CHERRY_PICKED_LABEL_PREFIX} -m '{CHERRY_PICKED_LABEL_PREFIX}: [{target_branch}] {commit_msg_striped}' -m 'cherry-pick {pull_request_url} into {target_branch}' -m 'requested-by {requested_by}'\"", ] rc, out, err = None, "", "" From 1e5440a017a26dd4e3b4cb2bf9fc4e0a8529adf0 Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Mon, 14 Oct 2024 17:44:25 +0300 Subject: [PATCH 16/17] Fix cherry-pick --- webhook_server_container/libs/github_api.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/webhook_server_container/libs/github_api.py b/webhook_server_container/libs/github_api.py index 160d1cf9..e94e381f 100644 --- a/webhook_server_container/libs/github_api.py +++ b/webhook_server_container/libs/github_api.py @@ -355,7 +355,7 @@ def _repo_data_from_config(self) -> None: primary_dict=repo_data, secondary_dict=config_data, key="tox-python-version", - return_on_none="python", + return_on_none=None, ) self.slack_webhook_url: str = get_value_from_dicts( primary_dict=repo_data, secondary_dict=config_data, key="slack_webhook_url" @@ -1054,7 +1054,8 @@ def _run_tox(self) -> None: return clone_repo_dir = f"{self.clone_repo_dir}-{uuid4()}" - cmd = f"uvx --python={self.tox_python_version} {TOX_STR} --workdir {clone_repo_dir} --root {clone_repo_dir} -c {clone_repo_dir}" + python_ver = f"--python={self.tox_python_version}" if self.tox_python_version else "" + cmd = f"uvx {python_ver} {TOX_STR} --workdir {clone_repo_dir} --root {clone_repo_dir} -c {clone_repo_dir}" _tox_tests = self.tox.get(self.pull_request_branch, "") if _tox_tests != "all": tests = _tox_tests.replace(" ", "") From e8c9baaab14c6a32cf4977f54107bb1da1dc6e52 Mon Sep 17 00:00:00 2001 From: Meni Yakove Date: Mon, 14 Oct 2024 17:55:33 +0300 Subject: [PATCH 17/17] dockerfile cleanup --- Dockerfile | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/Dockerfile b/Dockerfile index 348de3c9..c514163b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,31 +11,16 @@ ENV DATA_DIR="$HOME_DIR/data" ENV APP_DIR="$HOME_DIR/github-webhook-server" RUN dnf -y install dnf-plugins-core \ - && dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo \ && dnf -y update \ - && dnf -y install python3.8 \ - python3.9 \ - python3.10 \ - python3.11 \ - python3.12 \ - python3-pip \ + && dnf -y install \ git \ hub \ unzip \ - libcurl-devel \ gcc \ python3-devel \ - libffi-devel \ - docker-ce \ - docker-ce-cli \ - containerd.io \ - docker-buildx-plugin \ - docker-compose-plugin \ - slirp4netns \ && dnf clean all \ && rm -rf /var/cache /var/log/dnf* /var/log/yum.* -# RUN ln -s /usr/bin/python3 /usr/bin/python RUN mkdir -p $BIN_DIR \ && mkdir -p $APP_DIR \ @@ -64,20 +49,6 @@ RUN set -x \ && chmod +x $BIN_DIR/rosa \ && rm -rf $BIN_DIR/rosa-linux.tar.gz -# RUN python -m pip install --no-cache-dir pip --upgrade \ -# && python -m pip install --no-cache-dir poetry tox twine pre-commit -# -# RUN python3.8 -m ensurepip \ -# && python3.9 -m ensurepip \ -# && python3.10 -m ensurepip \ -# && python3.11 -m ensurepip \ -# && python3.12 -m ensurepip \ -# && python3.8 -m pip install tox \ -# && python3.9 -m pip install tox \ -# && python3.10 -m pip install tox \ -# && python3.11 -m pip install tox \ -# && python3.12 -m pip install tox - WORKDIR $APP_DIR RUN uv sync