metrics tracking with prometheus, addressing issue #7

Author: memalhot

.gitignore

@@ -4,3 +4,4 @@ __pycache__
 .coverage
 htmlcov
 *egg-info
+*venv*

batchtools/batchtools.py

@@ -10,7 +10,7 @@
 from .bq import GpuQueuesCommand
 from .br import CreateJobCommand
 from .bps import ListPodsCommand
-from .helpers import is_logged_in
+from .helpers import is_logged_in, is_on_project
 
 
 class BatchTools:
@@ -61,6 +61,10 @@ def main() -> None:
         sys.exit(
             "You are not logged in to the oc cli. Retrieve the token using 'oc login --web' or retrieving the login token from the openshift UI."
         )
+    if not is_on_project():
+        sys.exit(
+            "You are not on a project. For metric tracking, you must be on a project. Use oc project <project-name> to get on a project."
+        )
 
     app = BatchTools()
     app.run()

batchtools/bd.py

@@ -1,23 +1,26 @@
 import sys
 import argparse
 from typing import cast
-
 import openshift_client as oc
-
 from .basecommand import Command, override
 from .basecommand import SubParserFactory
-from .helpers import oc_delete
+from .helpers import oc_delete, is_kueue_managed_job
 
 
 class DeleteJobsCommand(Command):
     """
     batchtools bd [job-name [job-name ...]]
 
-    Delete specified Jobs, or all Jobs if none are specified.
+    Delete specified Kueue-managed GPU jobs, or all such jobs if none are specified.
+
+    Description:
+        Deletes only those Jobs that are both:
+          - named like your GPU jobs (name starts with 'job-'), and
+          - detected as Kueue-managed (via labels/Workload linkage).
     """
 
     name: str = "bd"
-    help: str = "Delete specified Jobs, or all if none are specified"
+    help: str = "Delete specified Kueue-managed GPU jobs, or all if none are specified"
 
     @classmethod
     @override
@@ -26,34 +29,39 @@ def build_parser(cls, subparsers: SubParserFactory):
         p.add_argument(
             "job_names",
             nargs="*",
-            help="Optional list of job names to delete",
+            help="Optional list of job names to delete (must be Kueue-managed)",
         )
         return p
 
     @staticmethod
     @override
     def run(args: argparse.Namespace):
         args = cast(DeleteJobsCommand, args)
-
         try:
             jobs = oc.selector("jobs").objects()
             if not jobs:
                 print("No jobs found.")
                 return
 
+            # only want to delete kueue jobs so filter for kueue jobs
+            kueue_gpu_jobs = [job for job in jobs if is_kueue_managed_job(job)]
+
+            if not kueue_gpu_jobs:
+                print("No Kueue-managed GPU jobs to delete.")
+                return
+
             if args.job_names:
-                # delete only specified jobs
-                existing = {job.model.metadata.name for job in jobs}
+                # if jobs are specified, only delete specified jobs
+                allowed = {job.model.metadata.name for job in kueue_gpu_jobs}
                 for name in args.job_names:
-                    if name not in existing:
-                        print(f"{name} does not exist; skipping.")
+                    if name not in allowed:
+                        print(f"{name} is not a Kueue-managed GPU job; skipping.")
                         continue
                     oc_delete("job", name)
                     print(f"Deleted job: {name}")
             else:
-                # delete all jobs
-                print("No job names provided -> deleting ALL jobs:\n")
-                for job in jobs:
+                print("No job names provided -> deleting all Kueue-managed GPU jobs:\n")
+                for job in kueue_gpu_jobs:
                     name = job.model.metadata.name
                     oc_delete("job", name)
                     print(f"Deleted job: {name}")

batchtools/bj.py

@@ -4,6 +4,7 @@
 import sys
 import openshift_client as oc
 
+from .helpers import is_kueue_managed_job
 from .basecommand import Command
 
 
@@ -35,8 +36,12 @@ def run(args: argparse.Namespace):
                 print("No jobs found.")
                 return
 
-            print(f"Found {len(jobs)} job(s):\n")
-            for job in jobs:
+            # filter only Kueue-managed jobs
+            managed = [job for job in jobs if is_kueue_managed_job(job)]
+
+            print(f"Found {len(managed)} job(s):\n")
+
+            for job in managed:
                 print(f"- {job.model.metadata.name}")
 
         except oc.OpenShiftPythonException as e: