Fix local repository config file loader

webhook_server/libs/config.py

@@ -1,26 +1,36 @@
 import os
 from typing import Any
 
+import github
 import yaml
 from simple_logger.logger import get_logger
 
 LOGGER = get_logger(name="config")
 
 
 class Config:
-    def __init__(self, repository: str | None = None, repository_full_name: str | None = None) -> None:
+    def __init__(
+        self,
+        repository: str | None = None,
+    ) -> None:
         self.data_dir: str = os.environ.get("WEBHOOK_SERVER_DATA_DIR", "/home/podman/data")
         self.config_path: str = os.path.join(self.data_dir, "config.yaml")
-        self.exists()
         self.repository = repository
-        self.repository_full_name = repository_full_name
+        self.exists()
+        self.repositories_exists()
 
     def exists(self) -> None:
         if not os.path.isfile(self.config_path):
             raise FileNotFoundError(f"Config file {self.config_path} not found")
 
+    def repositories_exists(self) -> None:
+        if not self.root_data.get("repositories"):
+            raise ValueError(f"Config {self.config_path} does not have `repositories`")
+
     @property
     def root_data(self) -> dict[str, Any]:
+        LOGGER.debug(f"Loading config file {self.config_path}")
+
         try:
             with open(self.config_path) as fd:
                 return yaml.safe_load(fd)
@@ -30,9 +40,31 @@ def root_data(self) -> dict[str, Any]:
 
     @property
     def repository_data(self) -> dict[str, Any]:
-        return self.root_data.get("repositories", {}).get(self.repository, {})
+        LOGGER.debug(f"Loading repository level config for repository {self.repository}")
+        return self.root_data["repositories"].get(self.repository, {})
+
+    def repository_local_data(self, github_api: github.Github, repository_full_name: str) -> dict[str, Any]:
+        LOGGER.debug(f"Loading local config for repository {repository_full_name}")
 
-    def get_value(self, value: str, return_on_none: Any = None) -> Any:
+        if self.repository and repository_full_name:
+            from webhook_server.utils.helpers import get_github_repo_api
+
+            try:
+                repo = get_github_repo_api(github_api=github_api, repository=repository_full_name)
+                _path = repo.get_contents(".github-webhook-server.yaml")
+                config_file = _path[0] if isinstance(_path, list) else _path
+                repo_config = yaml.safe_load(config_file.decoded_content)
+                LOGGER.debug(f"Repository {repository_full_name} config: {repo_config}")
+                return repo_config
+
+            except Exception as ex:
+                LOGGER.debug(f"Repository {repository_full_name} config file not found or error. {ex}")
+                return {}
+
+        LOGGER.debug("self.repository or self.repository_full_name is not defined")
+        return {}
+
+    def get_value(self, value: str, return_on_none: Any = None, extra_dict: dict[str, Any] | None = None) -> Any:
         """
         Get value from config
 
@@ -41,31 +73,15 @@ def get_value(self, value: str, return_on_none: Any = None) -> Any:
             2. Repository level global config file (config.yaml)
             3. Root level global config file (config.yaml)
         """
+        if extra_dict and extra_dict.get(value):
+            value = extra_dict[value]
+            if value is not None:
+                return value
 
-        for scope in (self.repository_local_data, self.repository_data, self.root_data):
+        for scope in (self.repository_data, self.root_data):
             if value in scope:
                 value_data = scope[value]
                 if value_data is not None:
                     return value_data
 
         return return_on_none
-
-    @property
-    def repository_local_data(self) -> dict[str, Any]:
-        if self.repository and self.repository_full_name:
-            from webhook_server.utils.helpers import get_api_with_highest_rate_limit, get_github_repo_api
-
-            github_api, _, _ = get_api_with_highest_rate_limit(config=self, repository_name=self.repository)
-
-            if github_api:
-                try:
-                    repo = get_github_repo_api(github_api=github_api, repository=self.repository_full_name)
-                    _path = repo.get_contents(".github-webhook-server.yaml")
-                    config_file = _path[0] if isinstance(_path, list) else _path
-                    return yaml.safe_load(config_file.decoded_content)
-
-                except Exception as ex:
-                    LOGGER.debug(f"Repository {self.repository_full_name} config file not found or error. {ex}")
-                    return {}
-
-        return {}

webhook_server/libs/github_api.py

@@ -112,13 +112,24 @@ def __init__(self, hook_data: dict[Any, Any], headers: Headers, logger: logging.
         self.owners_content: dict[str, Any] = {}
 
         self.config = Config(repository=self.repository_name)
-        self._repo_data_from_config()
+
+        if not self.config.repository:
+            raise RepositoryNotFoundError(f"Repository {self.repository_name} not found in config file")
+
+        # Get config without .github-webhook-server.yaml data
+        self._repo_data_from_config(repository_config={})
         self.github_api, self.token, self.api_user = get_api_with_highest_rate_limit(
             config=self.config, repository_name=self.repository_name
         )
 
         if self.github_api and self.token:
             self.repository = get_github_repo_api(github_api=self.github_api, repository=self.repository_full_name)
+            # Once we have a repository, we can get the config from .github-webhook-server.yaml
+            local_repository_config = self.config.repository_local_data(
+                github_api=self.github_api, repository_full_name=self.repository_full_name
+            )
+            # Call _repo_data_from_config() again to update self args from .github-webhook-server.yaml
+            self._repo_data_from_config(repository_config=local_repository_config)
 
         else:
             self.logger.error(f"Failed to get GitHub API and token for repository {self.repository_name}.")
@@ -298,19 +309,21 @@ def process_pull_request_check_run_webhook_data(self) -> None:
 
         return self.check_if_can_be_merged()
 
-    def _repo_data_from_config(self) -> None:
-        if not self.config.repository:
-            raise RepositoryNotFoundError(f"Repository {self.repository_name} not found in config file")
+    def _repo_data_from_config(self, repository_config: dict[str, Any]) -> None:
+        self.logger.debug(f"Read config for repository {self.repository_name}")
 
-        self.github_app_id: str = self.config.get_value(value="github-app-id")
-
-        self.pypi: dict[str, str] = self.config.get_value(value="pypi")
-        self.verified_job: bool = self.config.get_value(value="verified-job", return_on_none=True)
-        self.tox: dict[str, str] = self.config.get_value(value="tox")
-        self.tox_python_version: str = self.config.get_value(value="tox-python-version")
-        self.slack_webhook_url: str = self.config.get_value(value="slack_webhook_url")
+        self.github_app_id: str = self.config.get_value(value="github-app-id", extra_dict=repository_config)
+        self.pypi: dict[str, str] = self.config.get_value(value="pypi", extra_dict=repository_config)
+        self.verified_job: bool = self.config.get_value(
+            value="verified-job", return_on_none=True, extra_dict=repository_config
+        )
+        self.tox: dict[str, str] = self.config.get_value(value="tox", extra_dict=repository_config)
+        self.tox_python_version: str = self.config.get_value(value="tox-python-version", extra_dict=repository_config)
+        self.slack_webhook_url: str = self.config.get_value(value="slack_webhook_url", extra_dict=repository_config)
 
-        self.build_and_push_container: dict[str, Any] = self.config.get_value(value="container", return_on_none={})
+        self.build_and_push_container: dict[str, Any] = self.config.get_value(
+            value="container", return_on_none={}, extra_dict=repository_config
+        )
         if self.build_and_push_container:
             self.container_repository_username: str = self.build_and_push_container["username"]
             self.container_repository_password: str = self.build_and_push_container["password"]
@@ -321,17 +334,23 @@ def _repo_data_from_config(self) -> None:
             self.container_command_args: str = self.build_and_push_container.get("args", "")
             self.container_release: bool = self.build_and_push_container.get("release", False)
 
-        self.pre_commit: bool = self.config.get_value(value="pre-commit", return_on_none=False)
+        self.pre_commit: bool = self.config.get_value(
+            value="pre-commit", return_on_none=False, extra_dict=repository_config
+        )
 
         self.auto_verified_and_merged_users: list[str] = self.config.get_value(
-            value="auto-verified-and-merged-users", return_on_none=[]
+            value="auto-verified-and-merged-users", return_on_none=[], extra_dict=repository_config
         )
         self.can_be_merged_required_labels = self.config.get_value(
-            value="can-be-merged-required-labels", return_on_none=[]
+            value="can-be-merged-required-labels", return_on_none=[], extra_dict=repository_config
+        )
+        self.conventional_title: str = self.config.get_value(value="conventional-title", extra_dict=repository_config)
+        self.set_auto_merge_prs: list[str] = self.config.get_value(
+            value="set-auto-merge-prs", return_on_none=[], extra_dict=repository_config
+        )
+        self.minimum_lgtm: int = self.config.get_value(
+            value="minimum-lgtm", return_on_none=0, extra_dict=repository_config
         )
-        self.conventional_title: str = self.config.get_value(value="conventional-title")
-        self.set_auto_merge_prs: list[str] = self.config.get_value(value="set-auto-merge-prs", return_on_none=[])
-        self.minimum_lgtm: int = self.config.get_value(value="minimum-lgtm", return_on_none=0)
 
     def _get_pull_request(self, number: int | None = None) -> PullRequest:
         if number:

webhook_server/tests/test_branch_protection.py

@@ -26,11 +26,6 @@ def branch_protection_rules(request, mocker):
         return_value=root_data["repositories"][repo_name],
     )
 
-    mocker.patch(
-        f"{config_path}.repository_local_data",
-        new_callable=mocker.PropertyMock,
-        return_value={},
-    )
     return get_repo_branch_protection_rules(config=config)
 
 

webhook_server/tests/test_repo_data_from_config.py

@@ -1,5 +1,5 @@
 def test_repo_data_from_config_repository_found(process_github_webhook):
-    process_github_webhook._repo_data_from_config()
+    process_github_webhook._repo_data_from_config(repository_config={})
 
     assert process_github_webhook.repository_full_name == "my-org/test-repo"
     assert process_github_webhook.github_app_id == 123456

webhook_server/utils/helpers.py

@@ -159,6 +159,7 @@ def get_api_with_highest_rate_limit(
     logger.debug(msg)
 
     apis_and_tokens = get_apis_and_tokes_from_config(config=config)
+
     for _api, _token in apis_and_tokens:
         _api_user = _api.get_user().login
         rate_limit = _api.get_rate_limit()