Release/v0.2.12

mcs/__init__.py

@@ -1,4 +1,3 @@
 from mcs.api_client import APIClient
 from mcs.api import OnchainAPI
-from mcs.api import BucketAPI
-from mcs.contract import ContractClient
+from mcs.api import BucketAPI

mcs/api/bucket_api.py

@@ -1,9 +1,13 @@
+from aiohttp import request
 from mcs.api_client import APIClient
 from mcs.common.constants import *
 from hashlib import md5
 from queue import Queue
 import os, threading
 import urllib.request
+import logging
+import glob
+import shutil
 
 from mcs.common.utils import object_to_filename
 from mcs.object.bucket_storage import Bucket, File
@@ -16,41 +20,47 @@ def __init__(self, api_client=None):
         self.api_client = api_client
         self.MCS_API = api_client.MCS_API
         self.token = self.api_client.token
+        self.gateway = self.get_gateway()
 
     def list_buckets(self):
-        result = self.api_client._request_without_params(GET, BUCKET_LIST, self.MCS_API, self.token)
-        bucket_info_list = []
-        if result['status'] != 'success':
-            print("\033[31mError: " + result['message'] + "\033[0m" )
+        try:
+            result = self.api_client._request_without_params(GET, BUCKET_LIST, self.MCS_API, self.token)
+            bucket_info_list = []
+            data = result['data']
+            for bucket in data:
+                bucket_info: Bucket = Bucket(bucket)
+                bucket_info_list.append(bucket_info)
+            return bucket_info_list
+        except:
+            logging.error("\033[31m" + 'error' + "\033[0m")
             return
-        data = result['data']
-        for bucket in data:
-            bucket_info: Bucket = Bucket(bucket)
-            bucket_info_list.append(bucket_info)
-
-        # print(bucket_info_list)
-        return bucket_info_list
 
     def create_bucket(self, bucket_name):
         params = {'bucket_name': bucket_name}
-        result = self.api_client._request_with_params(POST, CREATE_BUCKET, self.MCS_API, params, self.token, None)
-        if result is None:
-            print("\033[31mError: This bucket already exists\033[0m")
-            return False
-        if result['status'] == 'success':
-            print("\033[32mBucket created successfully\033[0m")
-            return True
+        try:
+            result = self.api_client._request_with_params(POST, CREATE_BUCKET, self.MCS_API, params, self.token, None)
+            if result['status'] == 'success':
+                logging.info("\033[32mBucket created successfully\033[0m")
+                return True
+            else:
+                logging.error("\033[31m" + result['message'] + "\033[0m")
+        except:
+            logging.error("\033[31mThis bucket already exists\033[0m")
+
+        return False
 
     def delete_bucket(self, bucket_name):
-        bucket_id = self._get_bucket_id(bucket_name)
-        params = {'bucket_uid': bucket_id}
-        result = self.api_client._request_with_params(GET, DELETE_BUCKET, self.MCS_API, params, self.token, None)
-        if result is None:
-            print("\033[31mError: Can't find this bucket\033[0m")
-            return False
-        if result['status'] == 'success':
-            print("\033[32mBucket delete successfully\033[0m")
-            return True
+        try:
+            bucket_id = self._get_bucket_id(bucket_name)
+            params = {'bucket_uid': bucket_id}
+            result = self.api_client._request_with_params(GET, DELETE_BUCKET, self.MCS_API, params, self.token, None)
+            if result['status'] == 'success':
+                logging.info("\033[32mBucket delete successfully\033[0m")
+                return True
+        except:
+            if result is None:
+                logging.error("\033[31mCan't find this bucket\033[0m")
+        return False
 
     def get_bucket(self, bucket_name='', bucket_id=''):
         bucketlist = self.list_buckets()
@@ -66,48 +76,61 @@ def get_bucket(self, bucket_name='', bucket_id=''):
             for bucket in bucketlist:
                 if bucket.bucket_uid == bucket_id:
                     return bucket
-        print("\033[31mError: User does not have this bucket\033[0m")
+        logging.error("\033[31mUser does not have this bucket\033[0m")
         return None
 
     # object name
     def get_file(self, bucket_name, object_name):
-        prefix, file_name = object_to_filename(object_name)
-        file_list = self._get_full_file_list(bucket_name, prefix)
-        for file in file_list:
-            if file.name == file_name:
-                return file
-        print("\033[31mError: Can't find this object\033[0m")
-        return None
+        try:
+            bucket_id = self._get_bucket_id(bucket_name)
+            params = {"bucket_uid": bucket_id, "object_name": object_name}
+
+            result = self.api_client._request_with_params(GET, GET_FILE, self.MCS_API, params, self.token, None)
+
+            if result:
+                return File(result['data'], self.gateway)
+        except:
+            print('error')
+        return
 
     def create_folder(self, bucket_name, folder_name, prefix=''):
-        bucket_id = self._get_bucket_id(bucket_name)
-        params = {"file_name": folder_name, "prefix": prefix, "bucket_uid": bucket_id}
-        result = self.api_client._request_with_params(POST, CREATE_FOLDER, self.MCS_API, params, self.token, None)
-        if result['status'] == 'success':
-            print("\033[32mFolder created successfully\033[0m")
-            return True
-        else:
-            print("\033[31mError: " + result['message']+ "\033[0m")
-            return False
+        try:
+            bucket_id = self._get_bucket_id(bucket_name)
+            params = {"file_name": folder_name, "prefix": prefix, "bucket_uid": bucket_id}
+            result = self.api_client._request_with_params(POST, CREATE_FOLDER, self.MCS_API, params, self.token, None)
+            if result['status'] == 'success':
+                logging.info("\033[31mFolder created successfully\033[0m")
+                return True
+            else:
+                logging.error("\033[31m" + result['message'] + "\033[0m")
+                return False
+        except:
+            logging.error("\033[31mCan't create this folder")
+            return
 
     def delete_file(self, bucket_name, object_name):
-        prefix, file_name = object_to_filename(object_name)
-        file_list = self._get_full_file_list(bucket_name, prefix)
-        file_id = ''
-        for file in file_list:
-            if file.name == file_name:
-                file_id = file.id
-        params = {'file_id': file_id}
-        if file_id == '':
-            print("\033[31mError: Can't find the file\033[0m")
-            return False
-        result = self.api_client._request_with_params(GET, DELETE_FILE, self.MCS_API, params, self.token, None)
-        if result['status'] == 'success':
-            print("\033[32mFile delete successfully\033[0m")
-            return True
-        else:
-            print("\033[31mError: Can't delete the file\033[0m")
-            return False
+        try:
+            prefix, file_name = object_to_filename(object_name)
+            file_list = self._get_full_file_list(bucket_name, prefix)
+
+            file_id = ''
+            for file in file_list:
+                if file.name == file_name:
+                    file_id = file.id
+            params = {'file_id': file_id}
+            if file_id == '':
+                logging.error("\033[31mCan't find the file\033[0m")
+                return False
+            result = self.api_client._request_with_params(GET, DELETE_FILE, self.MCS_API, params, self.token, None)
+            if result['status'] == 'success':
+                logging.info("\033[32mFile delete successfully\033[0m")
+                return True
+            else:
+                logging.error("\033[31mCan't delete the file\033[0m")
+                return False
+        except:
+            logging.error("\033[31mCan't find this bucket\033[0m")
+            return
 
     def list_files(self, bucket_name, prefix='', limit='10', offset="0"):
         bucket_id = self._get_bucket_id(bucket_name)
@@ -117,27 +140,32 @@ def list_files(self, bucket_name, prefix='', limit='10', offset="0"):
             files = result['data']['file_list']
             file_list = []
             for file in files:
-                file_info: File = File(file)
+                file_info: File = File(file, self.gateway)
                 file_list.append(file_info)
             return file_list
         else:
-            print("\033[31mError: " + result['message'] + "\033[0m")
+            logging.error("\033[31m" + result['message'] + "\033[0m")
             return False
 
-
-    def upload_file(self, bucket_name, object_name, file_path):
+    def upload_file(self, bucket_name, object_name, file_path, replace=False):
         prefix, file_name = object_to_filename(object_name)
         bucket_id = self._get_bucket_id(bucket_name)
-        if os.stat(file_path).st_size == 0:
-            print("\033[31mError:File size cannot be 0\033[0m")
-            return None
+
+        # if os.stat(file_path).st_size == 0:
+        #     logging.error("\033[31mFile size cannot be 0\033[0m")
+        #     return None
+
         file_size = os.stat(file_path).st_size
         with open(file_path, 'rb') as file:
             file_hash = md5(file.read()).hexdigest()
         result = self._check_file(bucket_id, file_hash, file_name, prefix)
         if result is None:
-            print("\033[31mError:Cannot found bucket\033[0m")
+            logging.error("\033[31mCan't find this bucket\033[0m")
             return
+        # Replace file if already existed
+        if result['data']['file_is_exist'] and replace:
+            self.delete_file(bucket_name, object_name)
+            result = self._check_file(bucket_id, file_hash, file_name, prefix)
         if not (result['data']['file_is_exist']):
             if not (result['data']['ipfs_is_exist']):
                 with open(file_path, 'rb') as file:
@@ -158,37 +186,63 @@ def upload_file(self, bucket_name, object_name, file_path):
                 result = self._merge_file(bucket_id, file_hash, file_name, prefix)
             file_id = result['data']['file_id']
             file_info = self._get_file_info(file_id)
-            print("\033[32mFile upload successfully\033[0m")
+            self._create_folders(bucket_name, prefix)
+            logging.info("\033[32mFile upload successfully\033[0m")
             return file_info
-        print("\033[31mError:File already existed\033[0m")
+        logging.error("\033[31mFile already exists\033[0m")
         return None
+
+    def _create_folders(self, bucket_name, path):
+        bucket_id = self._get_bucket_id(bucket_name)
+        path, folder_name = object_to_filename(path)
+        while folder_name:
+            params = {"file_name": folder_name, "prefix": path, "bucket_uid": bucket_id}
+            self.api_client._request_with_params(POST, CREATE_FOLDER, self.MCS_API, params, self.token, None)
+            path, folder_name = object_to_filename(path)
+
+    def _upload_to_bucket(self, bucket_name, file_path, prefix=''):
+        if os.path.isdir(file_path):
+            return self.upload_folder(bucket_name, file_path, prefix)
+        else:
+            file_name = os.path.basename(file_path)
+            return self.upload_file(bucket_name, os.path.join(prefix, file_name), file_path)
+
+    def upload_folder(self, bucket_name, folder_path, prefix=''):
+        folder_name = os.path.basename(folder_path)
+        self.create_folder(bucket_name, folder_name, prefix)
+        res = []
+        files = os.listdir(folder_path)
+        for f in files:
+            f_path = os.path.join(folder_path, f)
+            upload = self._upload_to_bucket(bucket_name, f_path, os.path.join(prefix, folder_name))
+            res.append(upload)
+
+        return res
 
-    # def upload_folder(self, bucket_id, folder_path, prefix=''):
-    #     path = os.path.basename(folder_path)
-    #     folder_name = os.path.splitext(path)[0]
-    #     self.create_folder(folder_name, bucket_id, prefix)
-    #     files = os.listdir(folder_path)
-    #     success = []
-    #     for f in files:
-    #         f_path = os.path.join(folder_path, f)
-    #         if os.path.isdir(f_path):
-    #             success.extend(self.upload_folder(bucket_id, f_path, os.path.join(prefix, folder_name)))
-    #         else:
-    #             self.upload_to_bucket(bucket_id, f_path, os.path.join(prefix, folder_name))
-    #             time.sleep(0.5)
-    #             success.append(f_path)
-    #     return success
+    # def upload_ipfs_folder(self, bucket_name, object_name, folder_path):
+    #     folder_name = os.path.basename(object_name) or os.path.basename(folder_path)
+    #     prefix = os.path.normpath(os.path.dirname(object_name)) if os.path.dirname(object_name) else ''
+    #     bucket_uid = self._get_bucket_id(bucket_name)
+    #     files = self._read_files(folder_path, folder_name)
+    #     form_data = {"folder_name": folder_name, "prefix": prefix, "bucket_uid": bucket_uid}
+    #     res = self.api_client._request_with_params(POST, PIN_IPFS, self.MCS_API, form_data, self.token, files)
+    #     if res:
+    #         folder = (File(res["data"], self.gateway))
+    #         return folder
+    #     else:
+    #         logging.error("\033[31mIPFS Folder Upload Error\033[0m")
 
     def download_file(self, bucket_name, object_name, local_filename):
         file = self.get_file(bucket_name, object_name)
         if file is not None:
             ipfs_url = file.ipfs_url
             with open(local_filename, 'wb') as f:
-                data = urllib.request.urlopen(ipfs_url)
-                f.write(data.read())
-            print("\033[32mFile download successfully\033[0m")
+                if file.size > 0:
+                    data = urllib.request.urlopen(ipfs_url)
+                    f.write(data.read())
+            logging.info("\033[32mFile download successfully\033[0m")
             return True
-        print('\033[31mError: File does not exist\033[0m')
+        logging.error('\033[31mFile does not exist\033[0m')
         return False
 
     def _check_file(self, bucket_id, file_hash, file_name, prefix=''):
@@ -234,12 +288,33 @@ def _get_full_file_list(self, bucket_name, prefix=''):
                 self.api_client._request_with_params(GET, FILE_LIST, self.MCS_API, params, self.token, None)['data'][
                     'file_list']
             for file in result:
-                file_info: File = File(file)
+                file_info: File = File(file, self.gateway)
                 file_list.append(file_info)
         return file_list
 
     def _get_file_info(self, file_id):
         params = {'file_id': file_id}
         result = self.api_client._request_with_params(GET, FILE_INFO, self.MCS_API, params, self.token, None)
-        file_info = File(result['data'])
+        file_info = File(result['data'], self.gateway)
         return file_info
+
+    def get_gateway(self):
+        result = self.api_client._request_without_params(GET, GET_GATEWAY, self.MCS_API, self.token)
+        return 'https://' + result['data'][0]
+
+    def _read_files(self, root_folder, folder_name):
+        # Create an empty list to store the file tuples
+        file_dict = []
+
+        # Use glob to retrieve the file paths in the directory and its subdirectories
+        file_paths = glob.glob(os.path.join(root_folder, '**', '*'), recursive=True)
+
+        # Loop through each file path and read the contents of the file
+        for file_path in file_paths:
+            if os.path.isfile(file_path):
+                # Get the relative path from the root folder
+                upload_path = folder_name + "/" + os.path.relpath(file_path, root_folder)
+                file_dict.append(('files', (
+                    upload_path, open(file_path, 'rb'))))
+
+        return file_dict