Change to LFRic Macros scripts for git (#117)

james-bruten-mo · yaswant · web-flow · commit bd64f5e98c90 · 2025-12-05T10:22:12.000Z
* update copyright checker

* update copyright checker

* update copyright_checker

* add GitInfo class

* remove unnecessary function

* add gitinfo class

* add check for detached head

* initial commit for suite_report

* task tables done

* update '

* make executable

* update suite_report

* run black

* revert test_fcm_bdiff deletion

* add finally with error to import statement

* didn't want finally

* revert bdiff changes

* add check for writable suite_path

* re-copy bdiff branch

* add detached head unit test

* run black

* update for jules primary source

* add class docstring

* edit dependency table

* run black

* Update suite_report_git/suite_data.py

Co-authored-by: Yaswant Pradhan &lt;2984440+yaswant@users.noreply.github.com&gt;

* Update suite_report_git/suite_data.py

Co-authored-by: Yaswant Pradhan &lt;2984440+yaswant@users.noreply.github.com&gt;

* Update suite_report_git/suite_data.py

Co-authored-by: Yaswant Pradhan &lt;2984440+yaswant@users.noreply.github.com&gt;

* change to using urls

* url for local clones

* run black

* collapse failed

* start changes

* add get_git_sources

* split get_git_sources

* change lfric_macros to git

* remove print

* update source dirs

* code review comments

* run black

* make source dir first

* fix

* add umdp3_fixer

* add wrapper script

* make copy of fcm nightly testing

* update read_sources

* run black

* black

* update extract loc

* add type hinting

* fix checkout

* update github extraction

* add mkdir

* try other apprach

* change to tuple

* revert to init

* move files around

* update extract

* update extract

* update extract

* update extract

* update suite_report

* unsaved

* update suite_report

* update branch

* add option to use tokens

* print extraction

* syntax

* fix

* add print

* update if

* update prints

* test

* test

* test

* remove debug prints

* update umdp3 error message

* use shutil instead

* Update lfric_macros/apply_macros.py

Co-authored-by: Yaswant Pradhan &lt;2984440+yaswant@users.noreply.github.com&gt;

* Update lfric_macros/apply_macros.py

Co-authored-by: Yaswant Pradhan &lt;2984440+yaswant@users.noreply.github.com&gt;

* Update lfric_macros/apply_macros.py

Co-authored-by: Yaswant Pradhan &lt;2984440+yaswant@users.noreply.github.com&gt;

* Update lfric_macros/apply_macros.py

Co-authored-by: Yaswant Pradhan &lt;2984440+yaswant@users.noreply.github.com&gt;

* Update lfric_macros/apply_macros.py

Co-authored-by: Yaswant Pradhan &lt;2984440+yaswant@users.noreply.github.com&gt;

* change test for git url

* cr comments

* update

* added space to incorrect place

---------

Co-authored-by: Yaswant Pradhan &lt;2984440+yaswant@users.noreply.github.com&gt;
diff --git a/lfric_macros/apply_macros.py b/lfric_macros/apply_macros.py
@@ -14,9 +14,11 @@
 import ast
 import os
 import re
+import shlex
 import shutil
 import subprocess
 import tempfile
+import yaml
 import networkx as nx
 from collections import defaultdict
 
@@ -34,7 +36,7 @@ def run_command(command, shell=False):
         - result object from subprocess.run
     """
     if not shell:
-        command = command.split()
+        command = shlex.split(command)
     return subprocess.run(
         command,
         capture_output=True,
@@ -72,31 +74,29 @@ def check_environment():
 
 def get_root_path(wc_path):
     """
-    Given a path to a working copy, ensure the path and working copy are both
-    valid and return the path to the working copy root directory
+    Given a path to a git clone, ensure the path and clone are both
+    valid and return the path to the clone root directory
     Inputs:
-        - wc_path, command line argument to a working copy
+        - wc_path, command line argument to a clone
     Outputs:
-        - str, path to the top level of the apps working copy
+        - str, path to the top level of the apps clone
     """
 
-    # Run fcm info on the given path to ensure it is
-    command = f"fcm info {wc_path}"
+    command = f"git -C {wc_path} rev-parse --show-toplevel"
     result = run_command(command)
     if result.returncode:
         print(
-            "[WARN] - Could not find the fcm root path for the apps working copy. "
-            "Defaulting to assuming the provided path in the root path."
+            "[WARN] - Unable to locate the git clone root for the apps working copy."
+            "Assuming the given path is the root directory by default."
         )
         return wc_path
 
     # If no error, then search through output for the working copy root path
     # return the found path
-    for line in result.stdout.split("\n"):
-        if line.startswith("Working Copy Root Path"):
-            return line.split(":", 1)[1].strip()
+    if result.stdout:
+        return result.stdout.strip()
     raise Exception(
-        "Couldn't extract the Working Copy Root path from the output of the "
+        "Couldn't extract the Git Clone Root path from the output of the "
         f"command '{command}'"
     )
 
@@ -321,11 +321,10 @@ def parse_application_section(self, meta_dir):
 
     def get_dependency_paths(self, source, repo):
         """
-        Parse the core or jules command line arguments to get the path to a
-        working copy.
+        Parse the core or jules command line arguments to get the path to a git clone.
         If the source isn't defined, first populate the source by reading the
-        dependencies.sh file.
-        If the source is an fcm url check it out to a temporary location
+        dependencies.yaml file.
+        If the source is a remote GitHub source clone it to a temporary location
         Inputs:
             - source, str, The command line argument for the source. If not set
                            this will be None
@@ -335,9 +334,9 @@ def get_dependency_paths(self, source, repo):
             - str, The path to the source working copy to use
         """
 
-        # If source is None then read the dependencies.sh file for the source
+        # If source is None then read the dependencies.yaml file for the source
         if source is None:
-            source = self.read_dependencies(repo)
+            source, ref = self.read_dependencies(repo)
 
         # If the source exists as a path then return as is
         if os.path.exists(os.path.expanduser(source)):
@@ -347,76 +346,58 @@ def get_dependency_paths(self, source, repo):
             if os.path.exists(source_path):
                 return source_path
 
-        # Check that the source looks like an fcm keyword, raise an error if not
-        if "fcm:" not in source:
+        # Check that the source looks like a GitHub URL, raise an error if not
+        if "git@github.com:" not in source and "https://github.com/" not in source:
             raise Exception(
                 f"The {repo} source: {source}, was not found as a working copy "
-                "and does not look like an fcm url. Please check the source."
-                "If not set on the command then the dependencies.sh file is "
+                "and does not look like an GitHub URL. Please check the source."
+                "If not set on the command then the dependencies.yaml file is "
                 "being used."
             )
 
-        # Checkout the fcm source to a temporary location
-        source = self.fcm_temp_copy(source, repo)
+        # Checkout the git source to a temporary location
+        source = self.git_clone_temp(source, ref, repo)
         return source
 
     def read_dependencies(self, repo):
         """
-        Read through the dependencies.sh file for the source of the repo defined
-        by repo. Uses self.root_path to locate the dependencies.sh file.
+        Read through the dependencies.yaml file for the source of the repo defined
+        by repo. Uses self.root_path to locate the dependencies.yaml file.
         Inputs:
             - repo, str, Either "lfric_core" or "jules" depending on which
                          source is being found. The function will work with
                          other repos, but not intended to within this script.
         Outputs:
-            - str, The source as defined by the dependencies.sh file
+            - str, The source as defined by the dependencies.yaml file
         """
-        dependencies_path = os.path.join(self.root_path, "dependencies.sh")
-        source = ""
-        rev = ""
-        with open(dependencies_path, "r") as dependencies_file:
-            # Loop over lines in dependencies.sh for lines relevant to repo
-            for line in dependencies_file:
-                line = line.strip()
-                if line.startswith(f"export {repo}_rev"):
-                    rev = line.split("=")[1]
-                if line.startswith(f"export {repo}_sources"):
-                    source = line.split("=")[1]
-        # If source not set then default to trunk
-        if source == "":
-            # lfric_core doesn't match the url
-            if repo == "lfric_core":
-                source = "fcm:lfric.xm_tr"
-            else:
-                source = f"fcm:{repo}.xm_tr"
-        # If a revision set then append to source
-        # Defaults to the head of the source
-        # Only do this if it's an fcm url
-        if rev != "" and "fcm:" in source:
-            source = f"{source}@{rev}"
-        return source
+        dependencies_path = os.path.join(self.root_path, "dependencies.yaml")
+        with open(dependencies_path, "r") as f:
+            dependencies = yaml.safe_load(f)
 
-    def fcm_temp_copy(self, url, repo):
+        return dependencies[repo]["source"], dependencies[repo]["ref"]
+
+    def git_clone_temp(self, source, ref, repo):
         """
-        Given an fcm url as a source, checkout a working copy to a temp location
-        and return the path. Update self.temp_dirs with temporary directory path
+        Given a github URL, extract a temporary clone
         Inputs:
-            - url, str, An fcm url of the source
+            - source, str, A github URL
+            - ref, str, A git ref to checkout, None will result in default branch
             - repo, str, the name of the source being found
         Outputs:
             - str, The path to the temporary working copy
         """
 
-        print(f"Extracting {url} to a temporary directory")
+        print(f"Extracting {source} to a temporary directory")
         tempdir = tempfile.mkdtemp()
         self.temp_dirs[repo] = tempdir
-        command = f"fcm co {url} {tempdir}"
-        result = run_command(command)
-        if result.returncode:
-            raise Exception(
-                f"Failed to checkout from URL {url} into directory {tempdir} "
-                f"with error message:\n\n{result.stderr}"
-            )
+        commands = (f"git clone {source} {tempdir}", f"git -C {tempdir} checkout {ref}")
+        for command in commands:
+            result = run_command(command)
+            if result.returncode:
+                raise Exception(
+                    f"Failed to clone from {source} into directory {tempdir} "
+                    f"with error message:\n\n{result.stderr}"
+                )
         return tempdir
 
     ############################################################################
@@ -1237,16 +1218,16 @@ def parse_args():
         "--core",
         default=None,
         help="The LFRic Core source being used."
-        "Either a path to a working copy or an FCM URL."
-        "If not set, will be read from the dependencies.sh",
+        "Either a path to a working copy or a git source."
+        "If not set, will be read from the dependencies.yaml",
     )
     parser.add_argument(
         "-j",
         "--jules",
         default=None,
         help="The Jules source being used."
-        "Either a path to a working copy or an FCM URL."
-        "If not set, will be read from the dependencies.sh",
+        "Either a path to a working copy or a git source."
+        "If not set, will be read from the dependencies.yaml",
     )
     return parser.parse_args()
 
@@ -1273,7 +1254,7 @@ def apply_macros_main(tag, cname=None, version=None, apps=".", core=None, jules=
             banner_print("WARNING")
             print(
                 "Macros have been applied to apps in LFRic Core. A temporary "
-                "copy of the LFRic Core source given by the `dependencies.sh` "
+                "copy of the LFRic Core source given by the `dependencies.yaml` "
                 f"file is located at:\n{macro_object.core_source}\nEnsure you "
                 "have committed those changes back to the core branch."
             )
diff --git a/lfric_macros/check_macro_chains.py b/lfric_macros/check_macro_chains.py
@@ -103,8 +103,8 @@ def main():
     Main function of the program
     """
 
-    source_apps = os.path.join(os.environ["SOURCE_ROOT"], "apps")
-    source_core = os.path.join(os.environ["SOURCE_ROOT"], "core")
+    source_apps = os.path.join(os.environ["SOURCE_ROOT"], "lfric_apps")
+    source_core = os.path.join(os.environ["SOURCE_ROOT"], "lfric_core")
 
     macro_object = ApplyMacros(
         "vn0.0_t0", None, "vn0.0", source_apps, source_core, None
diff --git a/lfric_macros/release_lfric.py b/lfric_macros/release_lfric.py
@@ -22,6 +22,8 @@
 import re
 import socket
 import subprocess
+import shutil
+import shlex
 
 from apply_macros import (
     ApplyMacros,
@@ -54,7 +56,7 @@ def run_command(command, timelimit=120):
         - result object from subprocess.run
     """
     result = subprocess.run(
-        command.split(),
+        shlex.split(command),
         capture_output=True,
         text=True,
         timeout=timelimit,
@@ -81,20 +83,25 @@ def set_dependency_path(args):
     LFRic Core source
     """
 
-    print("[INFO] Updating dependencies.sh Core source")
+    print("[INFO] Updating dependencies.yaml Core source")
 
     hostname = socket.gethostname()
-    dep_path = os.path.join(args.apps, "dependencies.sh")
+    dep_path = os.path.join(args.apps, "dependencies.yaml")
     with open(dep_path) as f:
         lines = f.readlines()
+    in_core = False
     for i, line in enumerate(lines):
-        if line.strip().startswith("export lfric_core_rev"):
-            lines[i] = "export lfric_core_rev=\n"
-        if line.strip().startswith("export lfric_core_sources"):
-            lines[i] = (
-                "export lfric_core_sources="
-                f"{hostname}:{os.path.abspath(args.core)}\n"
-            )
+        if line.strip().startswith("lfric_core"):
+            in_core = True
+        elif in_core and "source:" in line:
+            prefix, _, _ = line.partition("source:")
+            line = f"{prefix}source: {hostname}:{os.path.abspath(args.core)}\n"
+        elif in_core and "ref:" in line:
+            prefix, _, _ = line.partition("ref:")
+            line = f"{prefix}ref:"
+        elif in_core:
+            break
+        lines[i] = line
     with open(dep_path, "w") as f:
         f.write("".join(x for x in lines))
 
@@ -216,8 +223,16 @@ def copy_head_meta(meta_dirs, args):
     for meta_dir in meta_dirs:
         head = os.path.join(meta_dir, "HEAD")
         new = os.path.join(meta_dir, args.version)
-        command = f"fcm cp {head} {new}"
-        result = run_command(command)
+        shutil.copytree(head, new)
+        if args.core in new:
+            new = new.removeprefix(args.core)
+            new = new.lstrip("/")
+            command = f"git -C {args.core} add {new}"
+        elif args.apps in new:
+            new = new.removeprefix(args.apps)
+            new = new.lstrip("/")
+            command = f"git -C {args.apps} add {new}"
+        _ = run_command(command)
 
 
 def update_meta_import_path(meta_dirs, args):
@@ -264,8 +279,18 @@ def copy_versions_files(meta_dirs, args):
     for meta_dir in meta_dirs:
         versions_file = os.path.join(meta_dir, "versions.py")
         upgrade_file = os.path.join(meta_dir, upgrade_name)
-        command = f"fcm cp {versions_file} {upgrade_file}"
-        result = run_command(command)
+        if not os.path.exists(versions_file):
+            raise FileNotFoundError(f"The file {versions_file} doesn't exist")
+        shutil.copyfile(versions_file, upgrade_file)
+        if args.core in upgrade_file:
+            upgrade_file = upgrade_file.removeprefix(args.core)
+            upgrade_file = upgrade_file.lstrip("/")
+            command = f"git -C {args.core} add {upgrade_file}"
+        elif args.apps in upgrade_file:
+            upgrade_file = upgrade_file.removeprefix(args.apps)
+            upgrade_file = upgrade_file.lstrip("/")
+            command = f"git -C {args.apps} add {upgrade_file}"
+        _ = run_command(command)
 
     return upgrade_name
 
@@ -321,8 +346,7 @@ def update_versions_file(meta_dirs, upgrade_name):
 
     for meta_dir in meta_dirs:
         versions_file = os.path.join(meta_dir, "versions.py")
-        command = f"cp {template_path} {versions_file}"
-        result = run_command(command)
+        shutil.copyfile(template_path, versions_file)
         add_new_import(versions_file, upgrade_name)
 
 
