From 7e36f2090014f46c5cd167f64146949995536fbf Mon Sep 17 00:00:00 2001 From: Nitai Heeb Date: Wed, 2 Apr 2025 12:46:51 +0200 Subject: [PATCH 1/7] Plugin selection --- src/hermes/commands/init/base.py | 44 ++++++++++++++++++--- src/hermes/commands/init/util/slim_click.py | 5 ++- src/hermes/commands/marketplace.py | 35 ++++++++++++++-- 3 files changed, 73 insertions(+), 11 deletions(-) diff --git a/src/hermes/commands/init/base.py b/src/hermes/commands/init/base.py index 738ec026..8b89f03a 100644 --- a/src/hermes/commands/init/base.py +++ b/src/hermes/commands/init/base.py @@ -19,6 +19,7 @@ from requests import HTTPError import hermes.commands.init.util.slim_click as sc +from hermes.commands import marketplace from hermes.commands.base import HermesCommand, HermesPlugin from hermes.commands.init.util import (connect_github, connect_gitlab, connect_zenodo, git_info) @@ -58,15 +59,10 @@ class HermesInitFolderInfo: def __init__(self): self.absolute_path: str = "" self.has_git_folder: bool = False - # self.has_multiple_remotes: bool = False - # self.git_remote_url: str = "" - # self.git_base_url: str = "" - # self.used_git_hoster: GitHoster = GitHoster.Empty self.has_hermes_toml: bool = False self.has_gitignore: bool = False self.has_citation_cff: bool = False self.has_readme: bool = False - # self.current_branch: str = "" self.current_dir: str = "" self.dir_list: list[str] = [] self.dir_folders: list[str] = [] @@ -180,6 +176,7 @@ def __init__(self, parser: argparse.ArgumentParser): } self.plugin_relevant_commands = ["harvest", "deposit"] self.builtin_plugins: dict[str: HermesPlugin] = get_builtin_plugins(self.plugin_relevant_commands) + self.selected_plugins: list[marketplace.PluginInfo] = [] def init_command_parser(self, command_parser: argparse.ArgumentParser) -> None: command_parser.add_argument('--template-branch', nargs=1, default="", @@ -217,7 +214,10 @@ def __call__(self, args: argparse.Namespace) -> None: self.test_initialization() sc.echo(f"Starting to initialize HERMES in {self.folder_info.absolute_path}") - sc.max_steps = 7 + sc.max_steps = 8 + + sc.next_step("Configure HERMES plugins") + self.choose_plugins() sc.next_step("Configure deposition platform and setup method") self.choose_deposit_platform() @@ -595,6 +595,38 @@ def connect_deposit_platform(self) -> None: connect_zenodo.setup(using_sandbox=True) self.create_zenodo_token() + def choose_plugins(self): + """User chooses the plugins he wants to use.""" + plugin_infos: list[marketplace.PluginInfo] = marketplace.get_plugin_infos() + plugins_builtin: list[marketplace.PluginInfo] = list(filter(lambda p: p.builtin, plugin_infos)) + plugins_available: list[marketplace.PluginInfo] = list(filter(lambda p: not p.builtin, plugin_infos)) + plugins_selected: list[marketplace.PluginInfo] = [] + sc.echo("The following plugins are already builtin:") + for info in plugins_builtin: + sc.echo(str(info), formatting=sc.Formats.OKGREEN) + sc.echo("") + while True: + if plugins_selected: + sc.echo("The following plugins are going to be installed:") + for info in plugins_selected: + sc.echo(str(info), formatting=sc.Formats.OKCYAN) + sc.echo("") + if plugins_available: + sc.echo("The following plugins are available for installation:") + for info in plugins_available: + sc.echo(str(info), formatting=sc.Formats.WARNING, no_log=True) + sc.echo("") + else: + self.selected_plugins = plugins_selected + break + choice = sc.choose("Do you want to add a plugin?", ["No"] + [p.name for p in plugins_available]) + if choice == 0: + self.selected_plugins = plugins_selected + break + else: + chosen_plugin = plugins_available.pop(choice - 1) + plugins_selected.append(chosen_plugin) + def no_git_setup(self, start_question: str = "") -> None: """Makes the init for a gitless project (basically just creating hermes.toml)""" if start_question == "": diff --git a/src/hermes/commands/init/util/slim_click.py b/src/hermes/commands/init/util/slim_click.py index 5b705a41..51568cf6 100644 --- a/src/hermes/commands/init/util/slim_click.py +++ b/src/hermes/commands/init/util/slim_click.py @@ -62,18 +62,19 @@ def get_log_type(self, default: int = logging.INFO) -> int: return default -def echo(text: str, formatting: Formats = Formats.EMPTY, log_as: int = logging.NOTSET): +def echo(text: str, formatting: Formats = Formats.EMPTY, log_as: int = logging.NOTSET, no_log: bool = False): """ Prints the text with the given formatting. If log_as is set or AUTO_LOG_ON_ECHO is true it gets logged as well. :param text: The printed text. :param formatting: You can use the Formats Enum to give the text a special color or formatting. :param log_as: Creates a log entry with the given text if this is set. + :param no_log: Never creates a log entry if True. """ # Get logging type from formatting if AUTO_LOG_ON_ECHO if AUTO_LOG_ON_ECHO and log_as == logging.NOTSET and text != "": log_as = formatting.get_log_type(logging.INFO) # Add text to log if there is a logger - if log_as != logging.NOTSET and default_file_logger: + if log_as != logging.NOTSET and default_file_logger and no_log == False: default_file_logger.log(log_as, text) # Format the text for the console if formatting != Formats.EMPTY: diff --git a/src/hermes/commands/marketplace.py b/src/hermes/commands/marketplace.py index 842955dc..c88f7fb6 100644 --- a/src/hermes/commands/marketplace.py +++ b/src/hermes/commands/marketplace.py @@ -96,6 +96,38 @@ def _sort_plugins_by_step(plugins: list[SchemaOrgSoftwareApplication]) -> dict[s return sorted_plugins +def _plugin_loc(_plugin: SchemaOrgSoftwareApplication) -> str: + return "builtin" if _plugin.is_part_of == schema_org_hermes else (_plugin.url or "") + +class PluginInfo: + def __init__(self): + self.name: str = "" + self.location: str = "" + self.step: str = "" + self.builtin: bool = True + def __str__(self): + return f"[{self.step}] {self.name} ({self.location})" + + +def get_plugin_infos() -> list[PluginInfo]: + response = requests.get(MARKETPLACE_URL, headers={"User-Agent": hermes_user_agent}) + response.raise_for_status() + parser = PluginMarketPlaceParser() + parser.feed(response.text) + infos: list[PluginInfo] = [] + if parser.plugins: + plugins_sorted = _sort_plugins_by_step(parser.plugins) + for step in plugins_sorted.keys(): + for plugin in plugins_sorted[step]: + info = PluginInfo() + info.name = plugin.name + info.step = step + info.location = _plugin_loc(plugin) + info.builtin = plugin.is_part_of == schema_org_hermes + infos.append(info) + return infos + + def main(): response = requests.get(MARKETPLACE_URL, headers={"User-Agent": hermes_user_agent}) response.raise_for_status() @@ -108,9 +140,6 @@ def main(): MARKETPLACE_URL + "." ) - def _plugin_loc(_plugin: SchemaOrgSoftwareApplication) -> str: - return "builtin" if _plugin.is_part_of == schema_org_hermes else (_plugin.url or "") - if parser.plugins: print() max_name_len = max(map(lambda plugin: len(plugin.name), parser.plugins)) From 59fa59ce9d105598ba071e7429c8d10c7b049d63 Mon Sep 17 00:00:00 2001 From: Nitai Heeb Date: Mon, 7 Apr 2025 17:08:34 +0200 Subject: [PATCH 2/7] Init Plugin integration --- src/hermes/commands/init/base.py | 136 +++++++++++++------- src/hermes/commands/init/util/slim_click.py | 11 +- src/hermes/commands/marketplace.py | 34 ++++- 3 files changed, 130 insertions(+), 51 deletions(-) diff --git a/src/hermes/commands/init/base.py b/src/hermes/commands/init/base.py index 8b89f03a..08a6767c 100644 --- a/src/hermes/commands/init/base.py +++ b/src/hermes/commands/init/base.py @@ -7,14 +7,15 @@ import os import re import sys +import traceback +import requests +import toml + from dataclasses import dataclass from enum import Enum, auto from importlib import metadata from pathlib import Path from urllib.parse import urljoin, urlparse - -import requests -import toml from pydantic import BaseModel from requests import HTTPError @@ -174,6 +175,18 @@ def __init__(self, parser: argparse.ArgumentParser): "deposit_extra_files": "", "push_branch": "main" } + self.hermes_toml_data = { + "harvest": { + "sources": ["cff"] + }, + "deposit": { + "target": "invenio_rdm", + "invenio_rdm": { + "site_url": "", + "access_right": "open" + } + } + } self.plugin_relevant_commands = ["harvest", "deposit"] self.builtin_plugins: dict[str: HermesPlugin] = get_builtin_plugins(self.plugin_relevant_commands) self.selected_plugins: list[marketplace.PluginInfo] = [] @@ -210,40 +223,51 @@ def __call__(self, args: argparse.Namespace) -> None: if args.template_branch != "": self.template_branch = args.template_branch - # Test if init is valid in current folder - self.test_initialization() + try: + # Test if init is valid in current folder + self.test_initialization() + + sc.echo(f"Starting to initialize HERMES in {self.folder_info.absolute_path}\n") + sc.max_steps = 8 - sc.echo(f"Starting to initialize HERMES in {self.folder_info.absolute_path}") - sc.max_steps = 8 + sc.next_step("Configure HERMES plugins") + self.choose_plugins() + self.integrate_plugins() - sc.next_step("Configure HERMES plugins") - self.choose_plugins() + sc.next_step("Configure deposition platform and setup method") + self.choose_deposit_platform() + self.integrate_deposit_platform() + self.choose_setup_method() - sc.next_step("Configure deposition platform and setup method") - self.choose_deposit_platform() - self.choose_setup_method() + sc.next_step("Configure HERMES behaviour") + self.choose_push_branch() + self.choose_deposit_files() - sc.next_step("Configure HERMES behaviour") - self.choose_push_branch() - self.choose_deposit_files() + sc.next_step("Create hermes.toml file") + self.create_hermes_toml() - sc.next_step("Create hermes.toml file") - self.create_hermes_toml() + sc.next_step("Create CITATION.cff file") + self.create_citation_cff() - sc.next_step("Create CITATION.cff file") - self.create_citation_cff() + sc.next_step("Create git CI files") + self.update_gitignore() + self.create_ci_template() - sc.next_step("Create git CI files") - self.update_gitignore() - self.create_ci_template() + sc.next_step("Connect with deposition platform") + self.connect_deposit_platform() - sc.next_step("Connect with deposition platform") - self.connect_deposit_platform() + sc.next_step("Connect with git hoster") + self.configure_git_project() - sc.next_step("Connect with git hoster") - self.configure_git_project() + sc.echo("\nHERMES is now initialized and ready to be used.\n", + formatting=sc.Formats.OKGREEN+sc.Formats.BOLD) - sc.echo("\nHERMES is now initialized and ready to be used.\n", formatting=sc.Formats.OKGREEN+sc.Formats.BOLD) + except Exception as e: + # More useful message on error + sc.echo(f"An error occurred during execution of HERMES init: {e}", + formatting=sc.Formats.FAIL+sc.Formats.BOLD) + sc.debug_info(traceback.format_exc()) + sys.exit(2) def test_initialization(self) -> None: """Test if init is possible and wanted. If not: sys.exit()""" @@ -284,6 +308,7 @@ def test_initialization(self) -> None: if self.git_remote: self.git_remote_url = git_info.get_remote_url(self.git_remote) self.git_hoster = get_git_hoster_from_url(self.git_remote_url) + # Abort with no remote else: sc.echo("Your git project does not have a remote. It is recommended for HERMES to " @@ -311,26 +336,12 @@ def test_initialization(self) -> None: sys.exit() def create_hermes_toml(self) -> None: - """Creates the hermes.toml file based on a dictionary""" - deposit_url = DepositPlatformUrls.get(self.deposit_platform) - default_values = { - "harvest": { - "sources": ["cff"] - }, - "deposit": { - "target": "invenio_rdm", - "invenio_rdm": { - "site_url": deposit_url, - "access_right": "open" - } - } - } - + """Creates the hermes.toml file based on a self.hermes_toml_data""" if (not self.folder_info.has_hermes_toml) \ or sc.confirm("Do you want to replace your `hermes.toml` with a new one?", default=True): with open('hermes.toml', 'w') as toml_file: # noinspection PyTypeChecker - toml.dump(default_values, toml_file) + toml.dump(self.hermes_toml_data, toml_file) sc.echo("`hermes.toml` was created.", formatting=sc.Formats.OKGREEN) def create_citation_cff(self) -> None: @@ -384,7 +395,6 @@ def create_ci_template(self) -> None: """Downloads and configures the ci workflow files using templates from the chosen template branch.""" match self.git_hoster: case GitHoster.GitHub: - # TODO Replace this later with the link to the real templates (not the feature branch) template_url = self.get_template_url("TEMPLATE_hermes_github_to_zenodo.yml") ci_file_folder = ".github/workflows" ci_file_name = "hermes_github.yml" @@ -429,10 +439,11 @@ def configure_ci_template(self, ci_file_path) -> None: parameters = list(set(re.findall(r'{%(.*?)%}', content))) for parameter in parameters: if parameter in self.ci_parameters: - content = content.replace(f'{{%{parameter}%}}', self.ci_parameters[parameter]) + value = str(self.ci_parameters[parameter]) + content = content.replace(f'{{%{parameter}%}}', value) else: - sc.echo(f"Warning: CI File Parameter {{%{parameter}%}} was not set.", - formatting=sc.Formats.WARNING) + sc.debug_info(f"CI File Parameter {{%{parameter}%}} was not set.", formatting=sc.Formats.WARNING) + content = content.replace(f'{{%{parameter}%}}', '') with open(ci_file_path, 'w') as file: file.write(content) @@ -572,6 +583,11 @@ def choose_deposit_platform(self) -> None: ) self.deposit_platform = deposit_platform_list[deposit_platform_index] + def integrate_deposit_platform(self) -> None: + """Makes changes to the toml data or something else based on the chosen deposit platform.""" + deposit_url = DepositPlatformUrls.get(self.deposit_platform) + self.hermes_toml_data["deposit"]["invenio_rdm"]["site_url"] = deposit_url + def choose_setup_method(self) -> None: """User chooses his desired setup method: Either preferring automatic (if available) or manual.""" setup_method_index = sc.choose( @@ -615,11 +631,15 @@ def choose_plugins(self): sc.echo("The following plugins are available for installation:") for info in plugins_available: sc.echo(str(info), formatting=sc.Formats.WARNING, no_log=True) + if info.abstract: + sc.echo("-> " + info.abstract, formatting=sc.Formats.ITALIC+sc.Formats.WARNING, no_log=True) sc.echo("") else: self.selected_plugins = plugins_selected break - choice = sc.choose("Do you want to add a plugin?", ["No"] + [p.name for p in plugins_available]) + no_text = "No further plugins needed" + choice = sc.choose("Do you want to add a plugin?", + [no_text] + [f"Add {p.name}" for p in plugins_available]) if choice == 0: self.selected_plugins = plugins_selected break @@ -627,6 +647,26 @@ def choose_plugins(self): chosen_plugin = plugins_available.pop(choice - 1) plugins_selected.append(chosen_plugin) + def integrate_plugins(self): + """ + Plugin installation is added to the ci-parameters. + Also for now we use this method to do custom plugin installation steps. + """ + for plugin_info in self.selected_plugins: + if not plugin_info.is_valid(): + sc.echo(f"Could not install plugin: {plugin_info.name}", formatting=sc.Formats.FAIL) + continue + pip_install = plugin_info.get_pip_install_command() + self.ci_parameters["pip_install_plugins_github"] = \ + self.ci_parameters.get("pip_install_plugins_github", "") + " - run: " + pip_install + "\n" + self.ci_parameters["pip_install_plugins_gitlab"] = \ + self.ci_parameters.get("pip_install_plugins_gitlab", "") + " - " + pip_install + "\n" + match plugin_info.name: + case "hermes-plugin-python": + self.hermes_toml_data["harvest"]["sources"].append("toml") + case "hermes-plugin-git": + self.hermes_toml_data["harvest"]["sources"].append("git") + def no_git_setup(self, start_question: str = "") -> None: """Makes the init for a gitless project (basically just creating hermes.toml)""" if start_question == "": diff --git a/src/hermes/commands/init/util/slim_click.py b/src/hermes/commands/init/util/slim_click.py index 51568cf6..f1b0f1fa 100644 --- a/src/hermes/commands/init/util/slim_click.py +++ b/src/hermes/commands/init/util/slim_click.py @@ -61,6 +61,9 @@ def get_log_type(self, default: int = logging.INFO) -> int: return logging.INFO return default + def wrap_around(self, text: str) -> str: + return self.get_ansi() + text + Formats.ENDC.get_ansi() + def echo(text: str, formatting: Formats = Formats.EMPTY, log_as: int = logging.NOTSET, no_log: bool = False): """ @@ -74,10 +77,12 @@ def echo(text: str, formatting: Formats = Formats.EMPTY, log_as: int = logging.N if AUTO_LOG_ON_ECHO and log_as == logging.NOTSET and text != "": log_as = formatting.get_log_type(logging.INFO) # Add text to log if there is a logger - if log_as != logging.NOTSET and default_file_logger and no_log == False: + if log_as != logging.NOTSET and default_file_logger and not no_log: default_file_logger.log(log_as, text) # Format the text for the console if formatting != Formats.EMPTY: + if Formats.ENDC.get_ansi() in text: + text = text.replace(Formats.ENDC.get_ansi(), Formats.ENDC.get_ansi() + formatting.get_ansi()) text = f"{formatting.get_ansi()}{text}{Formats.ENDC.get_ansi()}" # Print it if (log_as != logging.DEBUG) or PRINT_DEBUG: @@ -165,7 +170,9 @@ def next_step(description: str): def create_console_hyperlink(url: str, word: str) -> str: """Use this to have a consistent display of hyperlinks.""" - return f"\033]8;;{url}\033\\{word}\033]8;;\033\\" if USE_FANCY_HYPERLINKS else f"{word} ({url})" + if USE_FANCY_HYPERLINKS: + return f"\033]8;;{url}\033\\{word}\033]8;;\033\\" + return f"{word} ({url})" class ColorLogHandler(logging.Handler): diff --git a/src/hermes/commands/marketplace.py b/src/hermes/commands/marketplace.py index c88f7fb6..32804895 100644 --- a/src/hermes/commands/marketplace.py +++ b/src/hermes/commands/marketplace.py @@ -11,6 +11,7 @@ from pydantic import BaseModel, ConfigDict, Field from pydantic.alias_generators import to_camel +from hermes.commands.init.util import slim_click from hermes.utils import hermes_doi, hermes_user_agent MARKETPLACE_URL = "https://hermes.software-metadata.pub/marketplace" @@ -99,14 +100,43 @@ def _sort_plugins_by_step(plugins: list[SchemaOrgSoftwareApplication]) -> dict[s def _plugin_loc(_plugin: SchemaOrgSoftwareApplication) -> str: return "builtin" if _plugin.is_part_of == schema_org_hermes else (_plugin.url or "") + class PluginInfo: + """ + This class contains all the information about a plugin which are needed for the init-Command. + """ def __init__(self): self.name: str = "" self.location: str = "" self.step: str = "" self.builtin: bool = True + self.install_url: str = "" + self.abstract: str = "" + def __str__(self): - return f"[{self.step}] {self.name} ({self.location})" + step_text = f"[{self.step}]" + return f"{step_text} {slim_click.Formats.BOLD.wrap_around(self.name)} ({self.location})" + + def get_pip_install_command(self) -> str: + """ + Returns the pip install command which can be used to install the plugin. + Tries to extract the project name from the install_url (PyPI-URL) if possible. + Otherwise, it tries to use the location (Git-Project-URL) for the pip install command. + """ + if self.install_url and self.install_url.startswith("https://pypi.org/project/"): + project_name = self.install_url.rstrip("/").removeprefix("https://pypi.org/project/") + return f"pip install {project_name}" + if self.location and self.location.startswith(("https://", "git@", "ssh://")): + git_url = self.location.rstrip("/") + return f"pip install git+{git_url}" + return "" + + def is_valid(self) -> bool: + """ + Returns True if the plugin can be installed. Maybe we'll check the actual repository here later + to make sure that other things are valid too. + """ + return self.get_pip_install_command() != "" def get_plugin_infos() -> list[PluginInfo]: @@ -124,6 +154,8 @@ def get_plugin_infos() -> list[PluginInfo]: info.step = step info.location = _plugin_loc(plugin) info.builtin = plugin.is_part_of == schema_org_hermes + info.install_url = plugin.install_url + info.abstract = plugin.abstract infos.append(info) return infos From ff70fd37573688de8bc5f05e67f347479139dd95 Mon Sep 17 00:00:00 2001 From: Nitai Heeb Date: Wed, 23 Apr 2025 14:19:37 +0200 Subject: [PATCH 3/7] better error messages --- src/hermes/commands/init/base.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/hermes/commands/init/base.py b/src/hermes/commands/init/base.py index 08a6767c..2c5f4db3 100644 --- a/src/hermes/commands/init/base.py +++ b/src/hermes/commands/init/base.py @@ -158,6 +158,7 @@ class HermesInitCommand(HermesCommand): def __init__(self, parser: argparse.ArgumentParser): super().__init__(parser) self.folder_info: HermesInitFolderInfo = HermesInitFolderInfo() + self.hermes_was_already_installed: bool = False self.tokens: dict = {} self.setup_method: str = "" self.deposit_platform: DepositPlatform = DepositPlatform.Empty @@ -262,8 +263,13 @@ def __call__(self, args: argparse.Namespace) -> None: sc.echo("\nHERMES is now initialized and ready to be used.\n", formatting=sc.Formats.OKGREEN+sc.Formats.BOLD) + # Nice message on Ctrl+C + except KeyboardInterrupt: + sc.echo("HERMES init was aborted.", sc.Formats.WARNING) + sys.exit() + + # Useful message on error except Exception as e: - # More useful message on error sc.echo(f"An error occurred during execution of HERMES init: {e}", formatting=sc.Formats.FAIL+sc.Formats.BOLD) sc.debug_info(traceback.format_exc()) @@ -282,6 +288,7 @@ def test_initialization(self) -> None: # Look at the current folder self.refresh_folder_info() + self.hermes_was_already_installed = self.folder_info.has_hermes_toml # Abort if there is no git if not self.folder_info.has_git_folder: @@ -750,3 +757,7 @@ def choose_deposit_files(self) -> None: else: for file in self.folder_info.dir_list: sc.echo(f"\t\t{file}", formatting=sc.Formats.OKCYAN) + + def clean_up_files(self): + if not self.hermes_was_already_installed: + pass From 43d5f3b0dd0e656cd1f57697d79c87b9e023f566 Mon Sep 17 00:00:00 2001 From: Nitai Heeb Date: Mon, 28 Apr 2025 13:39:01 +0200 Subject: [PATCH 4/7] Automated clean up --- src/hermes/commands/init/base.py | 71 ++++++++++++++++++++++++------ src/hermes/commands/marketplace.py | 42 +++++++++--------- 2 files changed, 78 insertions(+), 35 deletions(-) diff --git a/src/hermes/commands/init/base.py b/src/hermes/commands/init/base.py index 2c5f4db3..f8d8448e 100644 --- a/src/hermes/commands/init/base.py +++ b/src/hermes/commands/init/base.py @@ -123,7 +123,10 @@ def string_in_file(file_path, search_string: str) -> bool: def get_builtin_plugins(plugin_commands: list[str]) -> dict[str: HermesPlugin]: - """Returns a list of installed HermesPlugins based on a list of related command names.""" + """ + Returns a list of installed HermesPlugins based on a list of related command names. + This is currently not used (we use the marketplace code instead) but maybe later. + """ plugins = {} for plugin_command_name in plugin_commands: entry_point_group = f"hermes.{plugin_command_name}" @@ -159,6 +162,7 @@ def __init__(self, parser: argparse.ArgumentParser): super().__init__(parser) self.folder_info: HermesInitFolderInfo = HermesInitFolderInfo() self.hermes_was_already_installed: bool = False + self.new_created_paths: list[Path] = [] self.tokens: dict = {} self.setup_method: str = "" self.deposit_platform: DepositPlatform = DepositPlatform.Empty @@ -260,12 +264,14 @@ def __call__(self, args: argparse.Namespace) -> None: sc.next_step("Connect with git hoster") self.configure_git_project() + self.clean_up_files(False) sc.echo("\nHERMES is now initialized and ready to be used.\n", formatting=sc.Formats.OKGREEN+sc.Formats.BOLD) # Nice message on Ctrl+C except KeyboardInterrupt: sc.echo("HERMES init was aborted.", sc.Formats.WARNING) + self.clean_up_files(True) sys.exit() # Useful message on error @@ -273,6 +279,7 @@ def __call__(self, args: argparse.Namespace) -> None: sc.echo(f"An error occurred during execution of HERMES init: {e}", formatting=sc.Formats.FAIL+sc.Formats.BOLD) sc.debug_info(traceback.format_exc()) + self.clean_up_files(True) sys.exit(2) def test_initialization(self) -> None: @@ -344,9 +351,11 @@ def test_initialization(self) -> None: def create_hermes_toml(self) -> None: """Creates the hermes.toml file based on a self.hermes_toml_data""" + hermes_toml_path = Path("hermes.toml") + self.mark_as_new_path(hermes_toml_path) if (not self.folder_info.has_hermes_toml) \ or sc.confirm("Do you want to replace your `hermes.toml` with a new one?", default=True): - with open('hermes.toml', 'w') as toml_file: + with open(hermes_toml_path, 'w') as toml_file: # noinspection PyTypeChecker toml.dump(self.hermes_toml_data, toml_file) sc.echo("`hermes.toml` was created.", formatting=sc.Formats.OKGREEN) @@ -377,17 +386,19 @@ def create_citation_cff(self) -> None: def update_gitignore(self) -> None: """Creates .gitignore if there is none and adds '.hermes' to it""" + gitignore_path = Path(".gitignore") + self.mark_as_new_path(gitignore_path) if not self.folder_info.has_gitignore: - open(".gitignore", 'w') + open(gitignore_path, 'w') sc.echo("A new `.gitignore` file was created.", formatting=sc.Formats.OKGREEN) self.refresh_folder_info() if self.folder_info.has_gitignore: - with open(".gitignore", "r") as file: + with open(gitignore_path, "r") as file: gitignore_lines = file.readlines() if any([line.startswith(".hermes") for line in gitignore_lines]): sc.echo("The `.gitignore` file already contains `.hermes/`") else: - with open(".gitignore", "a") as file: + with open(gitignore_path, "a") as file: file.write("# Ignoring all HERMES cache files\n") file.write(".hermes/\n") file.write("hermes.log\n") @@ -403,10 +414,12 @@ def create_ci_template(self) -> None: match self.git_hoster: case GitHoster.GitHub: template_url = self.get_template_url("TEMPLATE_hermes_github_to_zenodo.yml") - ci_file_folder = ".github/workflows" + ci_file_folder = Path(".github/workflows") ci_file_name = "hermes_github.yml" - Path(ci_file_folder).mkdir(parents=True, exist_ok=True) + self.mark_as_new_path(ci_file_folder) + ci_file_folder.mkdir(parents=True, exist_ok=True) ci_file_path = Path(ci_file_folder) / ci_file_name + self.mark_as_new_path(ci_file_path) download_file_from_url(template_url, ci_file_path) self.configure_ci_template(ci_file_path) sc.echo(f"GitHub CI: File was created at {ci_file_path}", formatting=sc.Formats.OKGREEN) @@ -414,8 +427,15 @@ def create_ci_template(self) -> None: gitlab_ci_template_url = self.get_template_url("TEMPLATE_hermes_gitlab_to_zenodo.yml") hermes_ci_template_url = self.get_template_url("hermes-ci.yml") gitlab_ci_path = Path(".gitlab-ci.yml") - Path("gitlab").mkdir(parents=True, exist_ok=True) - hermes_ci_path = Path("gitlab") / "hermes-ci.yml" + gitlab_folder_path = Path("gitlab") + hermes_ci_path = gitlab_folder_path / "hermes-ci.yml" + # Adding paths to our list + self.mark_as_new_path(gitlab_ci_path) + self.mark_as_new_path(gitlab_folder_path) + self.mark_as_new_path(hermes_ci_path) + # Creating the gitlab folder + gitlab_folder_path.mkdir(parents=True, exist_ok=True) + # Creating / updating gitlab-ci if gitlab_ci_path.exists(): if string_in_file(gitlab_ci_path, "hermes-ci.yml"): sc.echo(f"It seems like your {gitlab_ci_path} file is already configured for hermes.") @@ -426,6 +446,7 @@ def create_ci_template(self) -> None: download_file_from_url(gitlab_ci_template_url, gitlab_ci_path) sc.echo(f"GitLab CI: {gitlab_ci_path} was created.", formatting=sc.Formats.OKGREEN) self.configure_ci_template(gitlab_ci_path) + # Creating hermes-ci download_file_from_url(hermes_ci_template_url, hermes_ci_path) self.configure_ci_template(hermes_ci_path) @@ -618,7 +639,7 @@ def connect_deposit_platform(self) -> None: connect_zenodo.setup(using_sandbox=True) self.create_zenodo_token() - def choose_plugins(self): + def choose_plugins(self) -> None: """User chooses the plugins he wants to use.""" plugin_infos: list[marketplace.PluginInfo] = marketplace.get_plugin_infos() plugins_builtin: list[marketplace.PluginInfo] = list(filter(lambda p: p.builtin, plugin_infos)) @@ -654,7 +675,7 @@ def choose_plugins(self): chosen_plugin = plugins_available.pop(choice - 1) plugins_selected.append(chosen_plugin) - def integrate_plugins(self): + def integrate_plugins(self) -> None: """ Plugin installation is added to the ci-parameters. Also for now we use this method to do custom plugin installation steps. @@ -688,6 +709,7 @@ def no_git_setup(self, start_question: str = "") -> None: sc.next_step("Create CITATION.cff file") self.create_citation_cff() + self.clean_up_files(False) sc.echo("\nHERMES is now initialized (without git integration or CI/CD files).\n", formatting=sc.Formats.OKGREEN) @@ -758,6 +780,27 @@ def choose_deposit_files(self) -> None: for file in self.folder_info.dir_list: sc.echo(f"\t\t{file}", formatting=sc.Formats.OKCYAN) - def clean_up_files(self): - if not self.hermes_was_already_installed: - pass + def mark_as_new_path(self, path: Path, avoid_existing: bool = True) -> None: + """ + This method should be called directly BEFORE creating a new file in the given Path. + This way we can look if something already exists there to decide later-on if we want to delete it on abort. + """ + if (not avoid_existing) or (not path.exists()): + self.new_created_paths.append(path) + + def clean_up_files(self, aborted: bool) -> None: + """ + This gets called when init is finished (successfully or aborted). + It cleans up unwanted files (like .hermes folder) and everything new when aborted. + """ + os.remove(Path(".hermes")) + if aborted: + if not self.hermes_was_already_installed: + for path in reversed(self.new_created_paths): + try: + if path.is_dir(): + path.rmdir() + else: + os.remove(path) + except Exception as e: + sc.echo(f"Cleaning Warning: Could not remove {path}. ({e})") diff --git a/src/hermes/commands/marketplace.py b/src/hermes/commands/marketplace.py index a1c14142..c5c73cfa 100644 --- a/src/hermes/commands/marketplace.py +++ b/src/hermes/commands/marketplace.py @@ -153,27 +153,6 @@ def _plugin_loc(_plugin: SchemaOrgSoftwareApplication) -> str: return "builtin" if _plugin.is_part_of == schema_org_hermes else (_plugin.url or "") -def get_plugin_infos() -> list[PluginInfo]: - response = requests.get(MARKETPLACE_URL, headers={"User-Agent": hermes_user_agent}) - response.raise_for_status() - parser = PluginMarketPlaceParser() - parser.feed(response.text) - infos: list[PluginInfo] = [] - if parser.plugins: - plugins_sorted = _sort_plugins_by_step(parser.plugins) - for step in plugins_sorted.keys(): - for plugin in plugins_sorted[step]: - info = PluginInfo() - info.name = plugin.name - info.step = step - info.location = _plugin_loc(plugin) - info.builtin = plugin.is_part_of == schema_org_hermes - info.install_url = plugin.install_url - info.abstract = plugin.abstract - infos.append(info) - return infos - - def main(): response = requests.get(MARKETPLACE_URL, headers={"User-Agent": hermes_user_agent}) response.raise_for_status() @@ -248,3 +227,24 @@ def is_valid(self) -> bool: to make sure that other things are valid too. """ return self.get_pip_install_command() != "" + + +def get_plugin_infos() -> list[PluginInfo]: + response = requests.get(MARKETPLACE_URL, headers={"User-Agent": hermes_user_agent}) + response.raise_for_status() + parser = PluginMarketPlaceParser() + parser.feed(response.text) + infos: list[PluginInfo] = [] + if parser.plugins: + plugins_sorted = _sort_plugins_by_step(parser.plugins) + for step in plugins_sorted.keys(): + for plugin in plugins_sorted[step]: + info = PluginInfo() + info.name = plugin.name + info.step = step + info.location = _plugin_loc(plugin) + info.builtin = plugin.is_part_of == schema_org_hermes + info.install_url = plugin.install_url + info.abstract = plugin.abstract + infos.append(info) + return infos From e33c02317bd755d7d45b4a94ca3b07c1b721801d Mon Sep 17 00:00:00 2001 From: Nitai Heeb Date: Mon, 28 Apr 2025 14:04:17 +0200 Subject: [PATCH 5/7] Small fixes --- src/hermes/commands/init/base.py | 20 ++++++++++++++++---- src/hermes/commands/init/util/slim_click.py | 2 +- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/hermes/commands/init/base.py b/src/hermes/commands/init/base.py index f8d8448e..1c93dcaf 100644 --- a/src/hermes/commands/init/base.py +++ b/src/hermes/commands/init/base.py @@ -6,6 +6,7 @@ import logging import os import re +import shutil import sys import traceback import requests @@ -270,6 +271,7 @@ def __call__(self, args: argparse.Namespace) -> None: # Nice message on Ctrl+C except KeyboardInterrupt: + sc.echo("") sc.echo("HERMES init was aborted.", sc.Formats.WARNING) self.clean_up_files(True) sys.exit() @@ -416,10 +418,13 @@ def create_ci_template(self) -> None: template_url = self.get_template_url("TEMPLATE_hermes_github_to_zenodo.yml") ci_file_folder = Path(".github/workflows") ci_file_name = "hermes_github.yml" + ci_file_path = ci_file_folder / ci_file_name + # Adding paths to our list + self.mark_as_new_path(Path(".github")) self.mark_as_new_path(ci_file_folder) - ci_file_folder.mkdir(parents=True, exist_ok=True) - ci_file_path = Path(ci_file_folder) / ci_file_name self.mark_as_new_path(ci_file_path) + # Creating folder & ci file + ci_file_folder.mkdir(parents=True, exist_ok=True) download_file_from_url(template_url, ci_file_path) self.configure_ci_template(ci_file_path) sc.echo(f"GitHub CI: File was created at {ci_file_path}", formatting=sc.Formats.OKGREEN) @@ -723,7 +728,11 @@ def choose_push_branch(self) -> None: ] ) if push_choice == 0: - self.ci_parameters["push_branch"] = sc.answer("Enter target branch: ") + branch = sc.answer("Enter target branch: ") + self.ci_parameters["push_branch"] = branch + sc.echo(f"The HERMES pipeline will be activated when you push on {sc.Formats.BOLD.wrap_around(branch)}", + formatting=sc.Formats.OKGREEN) + sc.echo() elif push_choice == 1: sc.echo("Setting up triggering by tags is currently not implemented.", formatting=sc.Formats.WARNING) sc.echo(f"You can visit {TUTORIAL_URL} to set it up manually later-on.", formatting=sc.Formats.WARNING) @@ -793,7 +802,10 @@ def clean_up_files(self, aborted: bool) -> None: This gets called when init is finished (successfully or aborted). It cleans up unwanted files (like .hermes folder) and everything new when aborted. """ - os.remove(Path(".hermes")) + sc.echo("Cleaning unused files...") + hidden_hermes_path = Path(".hermes") + if hidden_hermes_path.exists() and hidden_hermes_path.is_dir(): + shutil.rmtree(hidden_hermes_path) if aborted: if not self.hermes_was_already_installed: for path in reversed(self.new_created_paths): diff --git a/src/hermes/commands/init/util/slim_click.py b/src/hermes/commands/init/util/slim_click.py index f1b0f1fa..2f72626f 100644 --- a/src/hermes/commands/init/util/slim_click.py +++ b/src/hermes/commands/init/util/slim_click.py @@ -65,7 +65,7 @@ def wrap_around(self, text: str) -> str: return self.get_ansi() + text + Formats.ENDC.get_ansi() -def echo(text: str, formatting: Formats = Formats.EMPTY, log_as: int = logging.NOTSET, no_log: bool = False): +def echo(text: str = "", formatting: Formats = Formats.EMPTY, log_as: int = logging.NOTSET, no_log: bool = False): """ Prints the text with the given formatting. If log_as is set or AUTO_LOG_ON_ECHO is true it gets logged as well. :param text: The printed text. From c93c8e0fceaf0a8680a28ff385db224c3b201072 Mon Sep 17 00:00:00 2001 From: Nitai Heeb Date: Tue, 29 Apr 2025 12:42:19 +0200 Subject: [PATCH 6/7] Clean up marketplace.py --- src/hermes/commands/marketplace.py | 32 +++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/hermes/commands/marketplace.py b/src/hermes/commands/marketplace.py index c5c73cfa..680158a3 100644 --- a/src/hermes/commands/marketplace.py +++ b/src/hermes/commands/marketplace.py @@ -96,6 +96,11 @@ def handle_data(self, data): plugin = SchemaOrgSoftwareApplication.model_validate_json(data) self.plugins.append(plugin) + def parse_plugins_from_url(self, url: str = MARKETPLACE_URL, user_agent: str = hermes_user_agent): + response = requests.get(url, headers={"User-Agent": user_agent}) + response.raise_for_status() + self.feed(response.text) + @cache def _doi_is_version_of_concept_doi(doi: str, concept_doi: str) -> bool: @@ -150,28 +155,22 @@ def _sort_plugins_by_step(plugins: list[SchemaOrgSoftwareApplication]) -> dict[s def _plugin_loc(_plugin: SchemaOrgSoftwareApplication) -> str: - return "builtin" if _plugin.is_part_of == schema_org_hermes else (_plugin.url or "") + return ( + "builtin" + if _is_hermes_reference(_plugin.is_part_of) + else (_plugin.url or "") + ) def main(): - response = requests.get(MARKETPLACE_URL, headers={"User-Agent": hermes_user_agent}) - response.raise_for_status() - parser = PluginMarketPlaceParser() - parser.feed(response.text) + parser.parse_plugins_from_url(MARKETPLACE_URL, hermes_user_agent) print( "A detailed list of available plugins can be found on the HERMES website at", MARKETPLACE_URL + "." ) - def _plugin_loc(_plugin: SchemaOrgSoftwareApplication) -> str: - return ( - "builtin" - if _is_hermes_reference(_plugin.is_part_of) - else (_plugin.url or "") - ) - if parser.plugins: print() max_name_len = max(map(lambda plugin: len(plugin.name), parser.plugins)) @@ -230,10 +229,11 @@ def is_valid(self) -> bool: def get_plugin_infos() -> list[PluginInfo]: - response = requests.get(MARKETPLACE_URL, headers={"User-Agent": hermes_user_agent}) - response.raise_for_status() + """ + Returns a List of PluginInfos which are meant to be used by the init-command. + """ parser = PluginMarketPlaceParser() - parser.feed(response.text) + parser.parse_plugins_from_url(MARKETPLACE_URL, hermes_user_agent) infos: list[PluginInfo] = [] if parser.plugins: plugins_sorted = _sort_plugins_by_step(parser.plugins) @@ -243,7 +243,7 @@ def get_plugin_infos() -> list[PluginInfo]: info.name = plugin.name info.step = step info.location = _plugin_loc(plugin) - info.builtin = plugin.is_part_of == schema_org_hermes + info.builtin = info.location == "builtin" info.install_url = plugin.install_url info.abstract = plugin.abstract infos.append(info) From 8684a42e6c02889b8a8190be70db5d1defdecd6b Mon Sep 17 00:00:00 2001 From: Nitai Heeb Date: Wed, 30 Apr 2025 12:14:05 +0200 Subject: [PATCH 7/7] init base isorted --- src/hermes/commands/init/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/hermes/commands/init/base.py b/src/hermes/commands/init/base.py index 1c93dcaf..5937eded 100644 --- a/src/hermes/commands/init/base.py +++ b/src/hermes/commands/init/base.py @@ -9,14 +9,14 @@ import shutil import sys import traceback -import requests -import toml - from dataclasses import dataclass from enum import Enum, auto from importlib import metadata from pathlib import Path from urllib.parse import urljoin, urlparse + +import requests +import toml from pydantic import BaseModel from requests import HTTPError