From 8aee41836b15391e519aeca44d2f807172545e4c Mon Sep 17 00:00:00 2001 From: rykci Date: Fri, 27 Jan 2023 02:35:57 -0500 Subject: [PATCH 1/5] upload folder function --- mcs/api/bucket_api.py | 49 ++++++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/mcs/api/bucket_api.py b/mcs/api/bucket_api.py index e6843c9..4f1fd43 100644 --- a/mcs/api/bucket_api.py +++ b/mcs/api/bucket_api.py @@ -4,6 +4,7 @@ from queue import Queue import os, threading import urllib.request +import time from mcs.common.utils import object_to_filename from mcs.object.bucket_storage import Bucket, File @@ -163,21 +164,39 @@ def upload_file(self, bucket_name, object_name, file_path): print("\033[31mError:File already existed\033[0m") return None - # def upload_folder(self, bucket_id, folder_path, prefix=''): - # path = os.path.basename(folder_path) - # folder_name = os.path.splitext(path)[0] - # self.create_folder(folder_name, bucket_id, prefix) - # files = os.listdir(folder_path) - # success = [] - # for f in files: - # f_path = os.path.join(folder_path, f) - # if os.path.isdir(f_path): - # success.extend(self.upload_folder(bucket_id, f_path, os.path.join(prefix, folder_name))) - # else: - # self.upload_to_bucket(bucket_id, f_path, os.path.join(prefix, folder_name)) - # time.sleep(0.5) - # success.append(f_path) - # return success + def upload_to_bucket(self, bucket_name, file_path, prefix=''): + if os.path.isdir(file_path): + return self.upload_folder(bucket_name, file_path, prefix) + else: + file_name = os.path.basename(file_path) + return self.upload_file(bucket_name, os.path.join(prefix,file_name), file_path) + + def upload_folder(self, bucket_name, folder_path, prefix=''): + # path = os.path.basename(folder_path) + # folder_name = os.path.splitext(path)[0] + # self.create_folder(bucket_name, folder_path, prefix) + # files = os.listdir(folder_path) + # success = [] + # for f in files: + # f_path = os.path.join(folder_path, f) + # if os.path.isdir(f_path): + # success.extend(self.upload_folder(bucket_name, f_path, os.path.join(prefix, folder_name))) + # else: + # self.upload_file(bucket_name, os.path.join(prefix, f_path), f_path,) + # time.sleep(0.5) + # success.append(f_path) + # return success + folder_name = os.path.basename(folder_path) + self.create_folder(bucket_name, folder_name, prefix) + res = [] + files = os.listdir(folder_path) + for f in files: + f_path = os.path.join(folder_path, f) + upload = self.upload_to_bucket(bucket_name, f_path, os.path.join(prefix, folder_name)) + res.append(upload) + + return res + def download_file(self, bucket_name, object_name, local_filename): file = self.get_file(bucket_name, object_name) From 4fabd97e4d1a5d3e32f610382360c53f77d0987b Mon Sep 17 00:00:00 2001 From: rykci Date: Fri, 27 Jan 2023 15:14:54 -0500 Subject: [PATCH 2/5] handle None error --- mcs/api/bucket_api.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/mcs/api/bucket_api.py b/mcs/api/bucket_api.py index 4f1fd43..87c9eb0 100644 --- a/mcs/api/bucket_api.py +++ b/mcs/api/bucket_api.py @@ -29,7 +29,6 @@ def list_buckets(self): bucket_info: Bucket = Bucket(bucket) bucket_info_list.append(bucket_info) - # print(bucket_info_list) return bucket_info_list def create_bucket(self, bucket_name): @@ -74,6 +73,9 @@ def get_bucket(self, bucket_name='', bucket_id=''): def get_file(self, bucket_name, object_name): prefix, file_name = object_to_filename(object_name) file_list = self._get_full_file_list(bucket_name, prefix) + if file_list is None: + print("\033[31mError: Can't find this bucket\033[0m") + return for file in file_list: if file.name == file_name: return file @@ -84,6 +86,9 @@ def create_folder(self, bucket_name, folder_name, prefix=''): bucket_id = self._get_bucket_id(bucket_name) params = {"file_name": folder_name, "prefix": prefix, "bucket_uid": bucket_id} result = self.api_client._request_with_params(POST, CREATE_FOLDER, self.MCS_API, params, self.token, None) + if result is None: + print("\033[31mError: Can't create this folder") + return if result['status'] == 'success': print("\033[32mFolder created successfully\033[0m") return True @@ -94,6 +99,9 @@ def create_folder(self, bucket_name, folder_name, prefix=''): def delete_file(self, bucket_name, object_name): prefix, file_name = object_to_filename(object_name) file_list = self._get_full_file_list(bucket_name, prefix) + if file_list is None: + print("\033[31mError: Can't find this bucket\033[0m") + return file_id = '' for file in file_list: if file.name == file_name: @@ -164,7 +172,7 @@ def upload_file(self, bucket_name, object_name, file_path): print("\033[31mError:File already existed\033[0m") return None - def upload_to_bucket(self, bucket_name, file_path, prefix=''): + def _upload_to_bucket(self, bucket_name, file_path, prefix=''): if os.path.isdir(file_path): return self.upload_folder(bucket_name, file_path, prefix) else: @@ -192,7 +200,7 @@ def upload_folder(self, bucket_name, folder_path, prefix=''): files = os.listdir(folder_path) for f in files: f_path = os.path.join(folder_path, f) - upload = self.upload_to_bucket(bucket_name, f_path, os.path.join(prefix, folder_name)) + upload = self._upload_to_bucket(bucket_name, f_path, os.path.join(prefix, folder_name)) res.append(upload) return res From 1e6b6dfb4efab09cb70322c2fdc77b4459e5d913 Mon Sep 17 00:00:00 2001 From: rykci Date: Fri, 27 Jan 2023 16:08:29 -0500 Subject: [PATCH 3/5] check chain_name --- mcs/api_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mcs/api_client.py b/mcs/api_client.py index 8464927..e9e04b9 100644 --- a/mcs/api_client.py +++ b/mcs/api_client.py @@ -29,9 +29,9 @@ def get_price_rate(self): def api_key_login(self): params = {'apikey': self.api_key, 'access_token': self.access_token, 'network': self.chain_name} - if params.get('apikey') == '' or params.get('access_token') == '': - print("\033[31mAPIkey or access token does not exist\033[0m") - return False + if params.get('apikey') == '' or params.get('access_token') == '' or params.get('chain_name') == '': + print("\033[31mAPIkey, access token, or chain name does not exist\033[0m") + return result = self._request_with_params(POST, APIKEY_LOGIN, self.MCS_API, params, None, None) if result is None: print("\033[31mRequest Error\033[0m") From e4c79a3cd082661c5949a8451c6d700043810700 Mon Sep 17 00:00:00 2001 From: rykci Date: Mon, 30 Jan 2023 12:06:28 -0500 Subject: [PATCH 4/5] replace print with logging --- mcs/api/bucket_api.py | 175 +++++++++++++++++++++--------------------- mcs/api_client.py | 23 +++--- 2 files changed, 98 insertions(+), 100 deletions(-) diff --git a/mcs/api/bucket_api.py b/mcs/api/bucket_api.py index 87c9eb0..a667219 100644 --- a/mcs/api/bucket_api.py +++ b/mcs/api/bucket_api.py @@ -1,10 +1,11 @@ +from aiohttp import request from mcs.api_client import APIClient from mcs.common.constants import * from hashlib import md5 from queue import Queue import os, threading import urllib.request -import time +import logging from mcs.common.utils import object_to_filename from mcs.object.bucket_storage import Bucket, File @@ -19,38 +20,45 @@ def __init__(self, api_client=None): self.token = self.api_client.token def list_buckets(self): - result = self.api_client._request_without_params(GET, BUCKET_LIST, self.MCS_API, self.token) - bucket_info_list = [] - if result['status'] != 'success': - print("\033[31mError: " + result['message'] + "\033[0m" ) + try: + result = self.api_client._request_without_params(GET, BUCKET_LIST, self.MCS_API, self.token) + bucket_info_list = [] + data = result['data'] + for bucket in data: + bucket_info: Bucket = Bucket(bucket) + bucket_info_list.append(bucket_info) + return bucket_info_list + except: + logging.error("\033[31m" + result['message'] + "\033[0m") return - data = result['data'] - for bucket in data: - bucket_info: Bucket = Bucket(bucket) - bucket_info_list.append(bucket_info) - - return bucket_info_list + def create_bucket(self, bucket_name): params = {'bucket_name': bucket_name} - result = self.api_client._request_with_params(POST, CREATE_BUCKET, self.MCS_API, params, self.token, None) - if result is None: - print("\033[31mError: This bucket already exists\033[0m") - return False - if result['status'] == 'success': - print("\033[32mBucket created successfully\033[0m") - return True + try: + result = self.api_client._request_with_params(POST, CREATE_BUCKET, self.MCS_API, params, self.token, None) + if result['status'] == 'success': + logging.info("\033[32mBucket created successfully\033[0m") + return True + else: + logging.error("\033[31m" + result['message'] + "\033[0m") + except: + logging.error("\033[31mThis bucket already exists\033[0m") + + return False def delete_bucket(self, bucket_name): - bucket_id = self._get_bucket_id(bucket_name) - params = {'bucket_uid': bucket_id} - result = self.api_client._request_with_params(GET, DELETE_BUCKET, self.MCS_API, params, self.token, None) - if result is None: - print("\033[31mError: Can't find this bucket\033[0m") - return False - if result['status'] == 'success': - print("\033[32mBucket delete successfully\033[0m") - return True + try: + bucket_id = self._get_bucket_id(bucket_name) + params = {'bucket_uid': bucket_id} + result = self.api_client._request_with_params(GET, DELETE_BUCKET, self.MCS_API, params, self.token, None) + if result['status'] == 'success': + logging.info("\033[32mBucket delete successfully\033[0m") + return True + except: + if result is None: + logging.error("\033[31mCan't find this bucket\033[0m") + return False def get_bucket(self, bucket_name='', bucket_id=''): bucketlist = self.list_buckets() @@ -66,57 +74,62 @@ def get_bucket(self, bucket_name='', bucket_id=''): for bucket in bucketlist: if bucket.bucket_uid == bucket_id: return bucket - print("\033[31mError: User does not have this bucket\033[0m") + logging.error("\033[31mUser does not have this bucket\033[0m") return None # object name def get_file(self, bucket_name, object_name): - prefix, file_name = object_to_filename(object_name) - file_list = self._get_full_file_list(bucket_name, prefix) - if file_list is None: - print("\033[31mError: Can't find this bucket\033[0m") + try: + prefix, file_name = object_to_filename(object_name) + file_list = self._get_full_file_list(bucket_name, prefix) + + for file in file_list: + if file.name == file_name: + return file + logging.error("\033[31mCan't find this object\033[0m") + return None + except: + logging.error("\033[31mCan't find this bucket\033[0m") return - for file in file_list: - if file.name == file_name: - return file - print("\033[31mError: Can't find this object\033[0m") - return None def create_folder(self, bucket_name, folder_name, prefix=''): - bucket_id = self._get_bucket_id(bucket_name) - params = {"file_name": folder_name, "prefix": prefix, "bucket_uid": bucket_id} - result = self.api_client._request_with_params(POST, CREATE_FOLDER, self.MCS_API, params, self.token, None) - if result is None: - print("\033[31mError: Can't create this folder") + try: + bucket_id = self._get_bucket_id(bucket_name) + params = {"file_name": folder_name, "prefix": prefix, "bucket_uid": bucket_id} + result = self.api_client._request_with_params(POST, CREATE_FOLDER, self.MCS_API, params, self.token, None) + if result['status'] == 'success': + logging.info("\033[31mFolder created successfully\033[0m") + return True + else: + logging.error("\033[31m" + result['message']+ "\033[0m") + return False + except: + logging.error("\033[31mCan't create this folder") return - if result['status'] == 'success': - print("\033[32mFolder created successfully\033[0m") - return True - else: - print("\033[31mError: " + result['message']+ "\033[0m") - return False def delete_file(self, bucket_name, object_name): - prefix, file_name = object_to_filename(object_name) - file_list = self._get_full_file_list(bucket_name, prefix) - if file_list is None: - print("\033[31mError: Can't find this bucket\033[0m") + try: + prefix, file_name = object_to_filename(object_name) + file_list = self._get_full_file_list(bucket_name, prefix) + + file_id = '' + for file in file_list: + if file.name == file_name: + file_id = file.id + params = {'file_id': file_id} + if file_id == '': + logging.error("\033[31mCan't find the file\033[0m") + return False + result = self.api_client._request_with_params(GET, DELETE_FILE, self.MCS_API, params, self.token, None) + if result['status'] == 'success': + logging.info("\033[32mFile delete successfully\033[0m") + return True + else: + logging.error("\033[31mCan't delete the file\033[0m") + return False + except: + logging.error("\033[31mCan't find this bucket\033[0m") return - file_id = '' - for file in file_list: - if file.name == file_name: - file_id = file.id - params = {'file_id': file_id} - if file_id == '': - print("\033[31mError: Can't find the file\033[0m") - return False - result = self.api_client._request_with_params(GET, DELETE_FILE, self.MCS_API, params, self.token, None) - if result['status'] == 'success': - print("\033[32mFile delete successfully\033[0m") - return True - else: - print("\033[31mError: Can't delete the file\033[0m") - return False def list_files(self, bucket_name, prefix='', limit='10', offset="0"): bucket_id = self._get_bucket_id(bucket_name) @@ -130,7 +143,7 @@ def list_files(self, bucket_name, prefix='', limit='10', offset="0"): file_list.append(file_info) return file_list else: - print("\033[31mError: " + result['message'] + "\033[0m") + logging.error("\033[31m" + result['message'] + "\033[0m") return False @@ -138,14 +151,14 @@ def upload_file(self, bucket_name, object_name, file_path): prefix, file_name = object_to_filename(object_name) bucket_id = self._get_bucket_id(bucket_name) if os.stat(file_path).st_size == 0: - print("\033[31mError:File size cannot be 0\033[0m") + logging.error("\033[31mFile size cannot be 0\033[0m") return None file_size = os.stat(file_path).st_size with open(file_path, 'rb') as file: file_hash = md5(file.read()).hexdigest() result = self._check_file(bucket_id, file_hash, file_name, prefix) if result is None: - print("\033[31mError:Cannot found bucket\033[0m") + logging.error("\033[31mCan't find this bucket\033[0m") return if not (result['data']['file_is_exist']): if not (result['data']['ipfs_is_exist']): @@ -167,9 +180,9 @@ def upload_file(self, bucket_name, object_name, file_path): result = self._merge_file(bucket_id, file_hash, file_name, prefix) file_id = result['data']['file_id'] file_info = self._get_file_info(file_id) - print("\033[32mFile upload successfully\033[0m") + logging.info("\033[32mFile upload successfully\033[0m") return file_info - print("\033[31mError:File already existed\033[0m") + logging.error("\033[31mFile already exists\033[0m") return None def _upload_to_bucket(self, bucket_name, file_path, prefix=''): @@ -180,20 +193,6 @@ def _upload_to_bucket(self, bucket_name, file_path, prefix=''): return self.upload_file(bucket_name, os.path.join(prefix,file_name), file_path) def upload_folder(self, bucket_name, folder_path, prefix=''): - # path = os.path.basename(folder_path) - # folder_name = os.path.splitext(path)[0] - # self.create_folder(bucket_name, folder_path, prefix) - # files = os.listdir(folder_path) - # success = [] - # for f in files: - # f_path = os.path.join(folder_path, f) - # if os.path.isdir(f_path): - # success.extend(self.upload_folder(bucket_name, f_path, os.path.join(prefix, folder_name))) - # else: - # self.upload_file(bucket_name, os.path.join(prefix, f_path), f_path,) - # time.sleep(0.5) - # success.append(f_path) - # return success folder_name = os.path.basename(folder_path) self.create_folder(bucket_name, folder_name, prefix) res = [] @@ -213,9 +212,9 @@ def download_file(self, bucket_name, object_name, local_filename): with open(local_filename, 'wb') as f: data = urllib.request.urlopen(ipfs_url) f.write(data.read()) - print("\033[32mFile download successfully\033[0m") + logging.info("\033[32mFile download successfully\033[0m") return True - print('\033[31mError: File does not exist\033[0m') + logging.error('\033[31mFile does not exist\033[0m') return False def _check_file(self, bucket_id, file_hash, file_name, prefix=''): diff --git a/mcs/api_client.py b/mcs/api_client.py index e9e04b9..83605b3 100644 --- a/mcs/api_client.py +++ b/mcs/api_client.py @@ -2,6 +2,7 @@ from mcs.common.params import Params import requests import json +import logging from mcs.common import utils, exceptions from mcs.common import constants as c from requests_toolbelt.multipart.encoder import MultipartEncoder, MultipartEncoderMonitor @@ -29,20 +30,18 @@ def get_price_rate(self): def api_key_login(self): params = {'apikey': self.api_key, 'access_token': self.access_token, 'network': self.chain_name} - if params.get('apikey') == '' or params.get('access_token') == '' or params.get('chain_name') == '': - print("\033[31mAPIkey, access token, or chain name does not exist\033[0m") - return - result = self._request_with_params(POST, APIKEY_LOGIN, self.MCS_API, params, None, None) - if result is None: - print("\033[31mRequest Error\033[0m") - return - if result['status'] != "success": - print("\033[31mError: " + result['message'] + ". \nPlease check your APIkey and access token, or " + # if params.get('apikey') == '' or params.get('access_token') == '' or params.get('chain_name') == '': + # logging.error("\033[31mAPIkey, access token, or chain name does not exist\033[0m") + # return + try: + result = self._request_with_params(POST, APIKEY_LOGIN, self.MCS_API, params, None, None) + self.token = result['data']['jwt_token'] + logging.info("\033[32mLogin successful\033[0m") + return self.token + except: + logging.error("\033[31m Please check your APIkey and access token, or " "check whether the current network environment corresponds to the APIkey.\033[0m") return - self.token = result['data']['jwt_token'] - print("\033[32mLogin successful\033[0m") - return self.token def _request(self, method, request_path, mcs_api, params, token, files=False): if method == c.GET: From 076b06335faaa3e194a6b097c13c73bcc886abc3 Mon Sep 17 00:00:00 2001 From: rykci Date: Thu, 2 Feb 2023 12:58:45 -0500 Subject: [PATCH 5/5] get file by object name --- mcs/api/bucket_api.py | 19 +++++++++---------- mcs/common/constants.py | 1 + 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/mcs/api/bucket_api.py b/mcs/api/bucket_api.py index a667219..a9cc733 100644 --- a/mcs/api/bucket_api.py +++ b/mcs/api/bucket_api.py @@ -29,7 +29,7 @@ def list_buckets(self): bucket_info_list.append(bucket_info) return bucket_info_list except: - logging.error("\033[31m" + result['message'] + "\033[0m") + logging.error("\033[31m" + 'error' + "\033[0m") return @@ -80,17 +80,16 @@ def get_bucket(self, bucket_name='', bucket_id=''): # object name def get_file(self, bucket_name, object_name): try: - prefix, file_name = object_to_filename(object_name) - file_list = self._get_full_file_list(bucket_name, prefix) + bucket_id = self._get_bucket_id(bucket_name) + params = {"bucket_uid": bucket_id, "object_name": object_name} + + result = self.api_client._request_with_params(GET, GET_FILE, self.MCS_API, params, self.token, None) - for file in file_list: - if file.name == file_name: - return file - logging.error("\033[31mCan't find this object\033[0m") - return None + if result: + return File(result['data']) except: - logging.error("\033[31mCan't find this bucket\033[0m") - return + print('error') + return def create_folder(self, bucket_name, folder_name, prefix=''): try: diff --git a/mcs/common/constants.py b/mcs/common/constants.py index 18b4f48..ecade99 100644 --- a/mcs/common/constants.py +++ b/mcs/common/constants.py @@ -29,6 +29,7 @@ UPLOAD_CHUNK = "/api/v2/oss_file/upload" MERGE_FILE = "/api/v2/oss_file/merge" FILE_LIST = "/api/v2/oss_file/get_file_list" +GET_FILE = "/api/v2/oss_file/get_file_by_object_name" # contract USDC_ABI = "ERC20.json" SWAN_PAYMENT_ABI = "SwanPayment.json"