Feature/submission envelop UUID for upload areas morphic redesign util

ait/commons/util/aws_client.py

@@ -1,7 +1,6 @@
 import json
 
 import boto3
-import botocore
 
 from ait.commons.util.aws_cognito_authenticator import AwsCognitoAuthenticator
 from ait.commons.util.settings import AWS_SECRET_NAME_AK_BUCKET, AWS_SECRET_NAME_SK_BUCKET, \

ait/commons/util/command/list.py

@@ -10,6 +10,10 @@ def print_area(k, area):
         p = area.get('perms') or ''
     print(p.ljust(3), end=' ')
 
+    if 'md5' in area:
+        p = area.get('md5') or ''
+    print(p.ljust(3), end=' ')
+
     if 'name' in area:
         n = area.get('name')
         print(f'{n}' if n else '', end=' ')
@@ -44,21 +48,23 @@ def run(self):
 
     def list_bucket_contents(self, selected_area, prefix=''):
         result = self.s3_cli.list_objects_v2(Bucket=selected_area, Delimiter='/', Prefix=prefix)
-
         # Folders
         dirs = result.get('CommonPrefixes', [])
 
         for d in dirs:
             k = d.get('Prefix')
-            print_area(k, {'key': k, 'perms': 'dir'})
+            print_area(k, {'key': k, 'md5': None, 'perms': 'dir'})
             self.list_bucket_contents(selected_area, prefix=k)
 
         # Files
         files = result.get('Contents', [])
 
         for f in files:
             k = f.get('Key')
-            print_area(k, {'key': k, 'perms': 'file'})
+            head_object_response = self.s3_cli.head_object(Bucket=selected_area, Key=k)
+            metadata = head_object_response.get('Metadata', {})
+            hash_md5 = metadata.get('md5', 'MD5 checksum not found')
+            print_area(k, {'key': k, 'md5': hash_md5, 'perms': 'file'})
 
     def list_bucket_contents_and_return(self, selected_area, prefix=''):
         """
@@ -71,26 +77,31 @@ def list_bucket_contents_and_return(self, selected_area, prefix=''):
         Returns:
         - A list of file keys in the bucket.
         """
-        file_keys = []
+        file_keys = []  # Initialize an empty list to store file keys.
 
+        # Define the recursive function to list bucket contents.
         def _list_bucket_contents(bucket, prefix):
+            # Call AWS S3 API to list objects with a specific prefix.
             result = self.s3_cli.list_objects_v2(Bucket=bucket, Delimiter='/', Prefix=prefix)
 
-            # Folders
+            # Handle directories (folders) first.
             dirs = result.get('CommonPrefixes', [])
             for d in dirs:
                 k = d.get('Prefix')
-                # print_area(k, {'key': k, 'perms': 'dir'})
+                # Recursively call the function to list contents of the subdirectory.
                 _list_bucket_contents(bucket, prefix=k)
 
-            # Files
+            # Handle files at the current prefix level.
             files = result.get('Contents', [])
             for f in files:
                 k = f.get('Key')
-                # print_area(k, {'key': k, 'perms': 'file'})
+                # Add each file key to the list.
                 file_keys.append(k)
 
+        # Start the recursive process to list all contents from the given prefix.
         _list_bucket_contents(selected_area, prefix)
+
+        # Return the final list of all file keys found in the bucket.
         return file_keys
 
 

ait/commons/util/command/upload.py

@@ -1,3 +1,4 @@
+import hashlib
 import os
 import filetype
 
@@ -9,6 +10,16 @@
 from ait.commons.util.progress_bar import ProgressBar
 
 
+def compute_md5(file_path):
+    """Compute the MD5 hash of the file."""
+    hash_md5 = hashlib.md5()
+
+    with open(file_path, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            hash_md5.update(chunk)
+    return hash_md5.hexdigest()
+
+
 class CmdUpload:
     """
     admin and user
@@ -21,6 +32,8 @@ def __init__(self, aws, args):
         self.files = []
 
     def upload_file(self, selected_area, data_file, destination_file):
+        hash_md5 = compute_md5(data_file)
+        print(f"MD5 hash of {data_file} is {hash_md5}")
 
         overwrite = getattr(self.args, 'o', False)
         file_size = os.path.getsize(data_file)
@@ -45,7 +58,9 @@ def upload_file(self, selected_area, data_file, destination_file):
             s3.Bucket(selected_area).upload_file(Filename=data_file,
                                                  Key=destination_file,
                                                  Callback=ProgressBar(target=data_file, total=file_size),
-                                                 ExtraArgs={'ContentType': content_type}
+                                                 ExtraArgs={'ContentType': content_type,
+                                                            'Metadata': {'md5': hash_md5}
+                                                            }
                                                  )
 
     def upload_files(self, data_files, prefix):

ait/commons/util/command/view.py

@@ -18,17 +18,17 @@ def __init__(self, args):
             print("Dataset is mandatory for view")
 
     def run(self):
-        fetched_dataset = self.provider_api.get_to_provider_api(f"{self.base_url}/datasets/{self.dataset}",
-                                                                self.access_token)
+        fetched_dataset = self.provider_api.get(f"{self.base_url}/datasets/{self.dataset}",
+                                                self.access_token)
         print(f"Dataset fetched successfully: {self.dataset}")
         print("Getting Biomaterials")
         biomaterials = fetched_dataset.get('biomaterials', [])
 
         for biomaterial in biomaterials:
             print(biomaterial)
 
-            fetched_biomaterial = self.provider_api.get_to_provider_api(f"{self.base_url}/biomaterials/{biomaterial}",
-                                                                        self.access_token)
+            fetched_biomaterial = self.provider_api.get(f"{self.base_url}/biomaterials/{biomaterial}",
+                                                        self.access_token)
             print(fetched_biomaterial)
 
         print("Getting Processes")
@@ -37,8 +37,8 @@ def run(self):
         for process in processes:
             print(process)
 
-            fetched_process = self.provider_api.get_to_provider_api(f"{self.base_url}/processes/{process}",
-                                                                    self.access_token)
+            fetched_process = self.provider_api.get(f"{self.base_url}/processes/{process}",
+                                                    self.access_token)
             print(fetched_process)
 
         print("Getting Data Files")
@@ -47,8 +47,8 @@ def run(self):
         for file in files:
             print(files)
 
-            fetched_file = self.provider_api.get_to_provider_api(f"{self.base_url}/files/{file}",
-                                                                 self.access_token)
+            fetched_file = self.provider_api.get(f"{self.base_url}/files/{file}",
+                                                 self.access_token)
             print(fetched_file)
 
         return True, "FETCHED SUCCESSFULLY"

ait/commons/util/provider_api_util.py

@@ -5,7 +5,7 @@ class APIProvider:
     def __init__(self, base_url):
         self.base_url = base_url
 
-    def send_request(self, method, url, access_token, params=None, data=None, data_type_in_hal_link=None):
+    def request(self, method, url, access_token, params=None, data=None, data_type_in_hal_link=None):
         """
         Sends an HTTP request to the specified URL with the given method.
 
@@ -74,18 +74,18 @@ def send_request(self, method, url, access_token, params=None, data=None, data_t
             # Return the JSON-parsed response data for other successful requests
             return response.json()
 
-    def put_to_provider_api(self, url, access_token):
-        return self.send_request('PUT', url, access_token)
+    def put(self, url, access_token):
+        return self.request('PUT', url, access_token)
 
-    def get_to_provider_api(self, url, access_token):
-        return self.send_request('GET', url, access_token)
+    def get(self, url, access_token):
+        return self.request('GET', url, access_token)
 
-    def delete_to_provider_api_including_linked_entities(self, url, access_token, delete_linked_entities=False):
+    def delete_with_relations(self, url, access_token, delete_linked_entities=False):
         params = {'deleteLinkedEntities': str(delete_linked_entities).lower()}
-        return self.send_request('DELETE', url, access_token, params=params)
+        return self.request('DELETE', url, access_token, params=params)
 
-    def delete_to_provider_api(self, url, access_token):
-        return self.send_request('DELETE', url, access_token)
+    def delete(self, url, access_token):
+        return self.request('DELETE', url, access_token)
 
-    def post_to_provider_api(self, url, data_type_in_hal_link, data, access_token):
-        return self.send_request('POST', url, access_token, data=data, data_type_in_hal_link=data_type_in_hal_link)
+    def post(self, url, data_type_in_hal_link, data, access_token):
+        return self.request('POST', url, access_token, data=data, data_type_in_hal_link=data_type_in_hal_link)

ait/commons/util/settings/morphic_util.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 
 NAME = 'morphic-util'
-VERSION = '1.0.0'
+VERSION = '1.0.1'
 DESC = 'CLI tool for submitting your analysis data and metadata'
 AUTHOR = 'dgupta'
 AUTHOR_EMAIL = 'dgupta@ebi.ac.uk'