Draft: Add pause on error, continue functionality

rebasebot/cli.py

@@ -1,5 +1,4 @@
 #!/usr/bin/python
-
 #    Copyright 2022 Red Hat, Inc.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
@@ -13,7 +12,6 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
-
 """This module parses CLI arguments for the Rebase Bot."""
 
 import argparse
@@ -22,7 +20,7 @@
 import sys
 import tempfile
 
-from rebasebot import bot, lifecycle_hooks
+from rebasebot import bot, lifecycle_hooks, resume_state
 from rebasebot.github import GithubAppProvider, GitHubBranch, parse_github_branch
 
 
@@ -184,6 +182,31 @@ def check_source_repo_args(namespace):
         required=False,
         help="When enabled, the bot will not create or update PR.",
     )
+    parser.add_argument(
+        "--pause-on-conflict",
+        action="store_true",
+        default=False,
+        required=False,
+        help=(
+            "Pause and persist resume state when a cherry-pick conflict needs manual resolution, "
+            "or when strict conflict policy detects dropped upstream content after a cherry-pick."
+        ),
+    )
+    parser.add_argument(
+        "--continue",
+        dest="continue_run",
+        action="store_true",
+        default=False,
+        required=False,
+        help="Continue a previously paused rebasebot run from the next saved step in the configured working directory.",
+    )
+    parser.add_argument(
+        "--retry-failed-step",
+        action="store_true",
+        default=False,
+        required=False,
+        help="When resuming a paused hook failure, retry the failed hook script instead of skipping to the next one.",
+    )
     parser.add_argument(
         "--tag-policy",
         default="none",
@@ -331,8 +354,19 @@ def rebasebot_run(args, slack_webhook, github_app_wrapper):
         args.working_dir = working_dir
         original_cwd = os.getcwd()
         try:
+            if args.continue_run is True and args.source_repo is not None:
+                try:
+                    persisted_state = resume_state.read_resume_state(working_dir)
+                except resume_state.ResumeStateError as e:
+                    logging.error(
+                        f"Error loading resume state before continue: {e}",
+                        exc_info=True,
+                    )
+                    sys.exit(1)
+                args.source = persisted_state.source.to_github_branch()
+
             try:
-                if args.source_repo is not None:
+                if args.source_repo is not None and args.continue_run is not True:
                     lifecycle_hooks.run_source_repo_hook(
                         args=args,
                         github_app_wrapper=github_app_wrapper,
@@ -373,6 +407,9 @@ def rebasebot_run(args, slack_webhook, github_app_wrapper):
                 hooks=hooks,
                 always_run_hooks=args.always_run_hooks,
                 title_prefix=args.title_prefix,
+                pause_on_conflict=args.pause_on_conflict is True,
+                continue_run=args.continue_run is True,
+                retry_failed_step=args.retry_failed_step is True,
             )
         finally:
             os.chdir(original_cwd)
@@ -402,10 +439,13 @@ def main():
         gh_user_token_path=args.github_user_token,
     )
 
-    if rebasebot_run(args, slack_webhook, github_app_wrapper):
-        sys.exit(0)
-    else:
-        sys.exit(1)
+    try:
+        if rebasebot_run(args, slack_webhook, github_app_wrapper):
+            sys.exit(0)
+        else:
+            sys.exit(1)
+    except bot.PausedRebaseException:
+        sys.exit(3)
 
 
 if __name__ == "__main__":

rebasebot/github.py

@@ -126,7 +126,9 @@ def __init__(
             self._app_credentials = GitHubAppCredentials(app_id=app_id, app_key=app_key, github_branch=dest_branch)
 
             self._cloner_app_credentials = GitHubAppCredentials(
-                app_id=cloner_id, app_key=cloner_key, github_branch=rebase_branch
+                app_id=cloner_id,
+                app_key=cloner_key,
+                github_branch=rebase_branch,
             )
 
     def get_app_token(self) -> str:

rebasebot/lifecycle_hooks.py

@@ -11,7 +11,6 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
-
 """This module manages user provided scripts that are executed during the rebase process."""
 
 import logging
@@ -31,7 +30,18 @@
 
 
 class LifecycleHookScriptException(Exception):
-    """LifecycleHookScriptException is a exception raised as a result of lifecycle hook script failure."""
+    """LifecycleHookScriptException is raised when a lifecycle hook script fails."""
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        script_index: int | None = None,
+        script_location: str | None = None,
+    ):
+        super().__init__(message)
+        self.script_index = script_index
+        self.script_location = script_location
 
 
 class LifecycleHook(Enum):
@@ -126,7 +136,14 @@ def _fetch_from_remote_git(
             raise ValueError(f"Failed to retrieve script from git reference {git_path}") from e
 
     def _fetch_from_github_api(
-        self, *, github, organization: str, name: str, git_repo_path_to_script: str, branch: str, script_file_path: str
+        self,
+        *,
+        github,
+        organization: str,
+        name: str,
+        git_repo_path_to_script: str,
+        branch: str,
+        script_file_path: str,
     ):
         """Fetches script from GitHub API."""
         try:
@@ -163,7 +180,8 @@ def fetch_script(self, temp_hook_dir: str, gitwd: git.Repo = None, github: Githu
             return
 
         remote_git_pattern_match = re.match(
-            "^git:(https://([^/]+)/([^/]+)/([^/]+))/([^:]+?):(.*)$", self.script_location
+            "^git:(https://([^/]+)/([^/]+)/([^/]+))/([^:]+?):(.*)$",
+            self.script_location,
         )
         local_git_pattern_match = re.match("^git:([^:]+):([^:]+)$", self.script_location)
 
@@ -238,7 +256,8 @@ def __call__(self, cwd: str = None) -> LifecycleHookScriptResult:
 
 def _fetch_file_from_github(github, organization, name, branch, git_repo_path_to_script) -> Contents:
     return github.github_cloner_app.repository(owner=organization, repository=name).file_contents(
-        git_repo_path_to_script, ref=branch
+        git_repo_path_to_script,
+        ref=branch,
     )
 
 
@@ -331,9 +350,18 @@ def fetch_hook_scripts(self, gitwd: git.Repo, github_app_provider: GithubAppProv
             for script in hooks:
                 script.fetch_script(temp_hook_dir=self.tmp_hook_scripts_dir, gitwd=gitwd, github=github_app_provider)
 
-    def execute_scripts_for_hook(self, hook: LifecycleHook):
-        """Executes all scripts in the given lifecycle hook."""
-        for script in self.hooks.get(hook, []):
+    def get_scripts_for_hook(self, hook: LifecycleHook) -> list[LifecycleHookScript]:
+        """Returns the configured scripts for the given lifecycle hook."""
+        return self.hooks.get(hook, [])
+
+    def get_script_locations_for_hook(self, hook: LifecycleHook) -> list[str]:
+        """Returns configured script locations for the given lifecycle hook."""
+        return [script.script_location for script in self.get_scripts_for_hook(hook)]
+
+    def execute_scripts_for_hook(self, hook: LifecycleHook, start_index: int = 0):
+        """Executes hook scripts starting at the given index."""
+        scripts = self.get_scripts_for_hook(hook)
+        for index, script in enumerate(scripts[start_index:], start=start_index):
             logging.info(f"Running {hook} lifecycle hook {script}")
             try:
                 result = script(cwd=self.working_dir)
@@ -347,4 +375,8 @@ def execute_scripts_for_hook(self, hook: LifecycleHook):
             except subprocess.CalledProcessError as err:
                 logging.error(f"Script {script} failed with exit code {err.returncode}")
                 message = f"{hook} script {script} failed with exit-code {err.returncode}"
-                raise LifecycleHookScriptException(message) from err
+                raise LifecycleHookScriptException(
+                    message,
+                    script_index=index,
+                    script_location=script.script_location,
+                ) from err