From a5ea9869f4042238a70d66ffc93910f74c92dd33 Mon Sep 17 00:00:00 2001 From: synicix Date: Wed, 9 Dec 2020 22:20:44 -0600 Subject: [PATCH 1/8] Update exception to more general form. Need to find out exactly what error it is to turn it back to specifics --- datajoint/s3.py | 7 ++++--- tests/__init__.py | 12 ++++++++---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/datajoint/s3.py b/datajoint/s3.py index 72e0f4f06..f16e5f0e1 100644 --- a/datajoint/s3.py +++ b/datajoint/s3.py @@ -37,7 +37,7 @@ def get(self, name): logger.debug('get: {}:{}'.format(self.bucket, name)) try: return self.client.get_object(self.bucket, str(name)).data - except minio.error.NoSuchKey: + except Exception as e: raise errors.MissingExternalFile('Missing s3 key %s' % name) from None def fget(self, name, local_filepath): @@ -59,7 +59,8 @@ def exists(self, name): logger.debug('exists: {}:{}'.format(self.bucket, name)) try: self.client.stat_object(self.bucket, str(name)) - except minio.error.NoSuchKey: + except Exception as e: + print(e) return False return True @@ -67,7 +68,7 @@ def get_size(self, name): logger.debug('get_size: {}:{}'.format(self.bucket, name)) try: return self.client.stat_object(self.bucket, str(name)).size - except minio.error.NoSuchKey: + except Exception as e: raise errors.MissingExternalFile from None def remove_object(self, name): diff --git a/tests/__init__.py b/tests/__init__.py index bb0b814c4..7ff8adaec 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -136,7 +136,9 @@ def setup_package(): region = "us-east-1" try: minioClient.make_bucket(S3_MIGRATE_BUCKET, location=region) - except minio.error.BucketAlreadyOwnedByYou: + except Exception as e: + print(e) + print('Minio make_bucket failed') pass pathlist = Path(source).glob('**/*') @@ -149,7 +151,9 @@ def setup_package(): # Add S3 try: minioClient.make_bucket(S3_CONN_INFO['bucket'], location=region) - except minio.error.BucketAlreadyOwnedByYou: + except Exception as e: + print(e) + print('Minio make_bucket failed') pass # Add old File Content @@ -179,14 +183,14 @@ def teardown_package(): remove("dj_local_conf.json") # Remove old S3 - objs = list(minioClient.list_objects_v2( + objs = list(minioClient.list_objects( S3_MIGRATE_BUCKET, recursive=True)) objs = [minioClient.remove_object(S3_MIGRATE_BUCKET, o.object_name.encode('utf-8')) for o in objs] minioClient.remove_bucket(S3_MIGRATE_BUCKET) # Remove S3 - objs = list(minioClient.list_objects_v2(S3_CONN_INFO['bucket'], recursive=True)) + objs = list(minioClient.list_objects(S3_CONN_INFO['bucket'], recursive=True)) objs = [minioClient.remove_object(S3_CONN_INFO['bucket'], o.object_name.encode('utf-8')) for o in objs] minioClient.remove_bucket(S3_CONN_INFO['bucket']) From 780d57bc14819ea3710c1276ab43c2be9fae54a1 Mon Sep 17 00:00:00 2001 From: synicix Date: Wed, 9 Dec 2020 22:29:24 -0600 Subject: [PATCH 2/8] Update print(e) statements to avoid github action errors --- datajoint/s3.py | 2 ++ tests/__init__.py | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datajoint/s3.py b/datajoint/s3.py index f16e5f0e1..19f792eb2 100644 --- a/datajoint/s3.py +++ b/datajoint/s3.py @@ -38,6 +38,7 @@ def get(self, name): try: return self.client.get_object(self.bucket, str(name)).data except Exception as e: + print(e) raise errors.MissingExternalFile('Missing s3 key %s' % name) from None def fget(self, name, local_filepath): @@ -69,6 +70,7 @@ def get_size(self, name): try: return self.client.stat_object(self.bucket, str(name)).size except Exception as e: + print(e) raise errors.MissingExternalFile from None def remove_object(self, name): diff --git a/tests/__init__.py b/tests/__init__.py index 7ff8adaec..807e90930 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -138,7 +138,6 @@ def setup_package(): minioClient.make_bucket(S3_MIGRATE_BUCKET, location=region) except Exception as e: print(e) - print('Minio make_bucket failed') pass pathlist = Path(source).glob('**/*') @@ -153,7 +152,6 @@ def setup_package(): minioClient.make_bucket(S3_CONN_INFO['bucket'], location=region) except Exception as e: print(e) - print('Minio make_bucket failed') pass # Add old File Content From d489b474a9747f29e3879e419de7b2f458113a7e Mon Sep 17 00:00:00 2001 From: synicix Date: Wed, 9 Dec 2020 23:05:07 -0600 Subject: [PATCH 3/8] Added more specifc cases for error --- tests/__init__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 807e90930..a2ecc65e8 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -136,8 +136,7 @@ def setup_package(): region = "us-east-1" try: minioClient.make_bucket(S3_MIGRATE_BUCKET, location=region) - except Exception as e: - print(e) + except minio.error.S3Error: pass pathlist = Path(source).glob('**/*') @@ -150,8 +149,7 @@ def setup_package(): # Add S3 try: minioClient.make_bucket(S3_CONN_INFO['bucket'], location=region) - except Exception as e: - print(e) + except minio.error.S3Error: pass # Add old File Content From 78c208af6da7e9217a020f4056acbb9628123990 Mon Sep 17 00:00:00 2001 From: synicix Date: Thu, 10 Dec 2020 00:26:43 -0600 Subject: [PATCH 4/8] Change error handling to be more specific to minio.error.S3Error --- datajoint/s3.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/datajoint/s3.py b/datajoint/s3.py index 19f792eb2..92f044eb8 100644 --- a/datajoint/s3.py +++ b/datajoint/s3.py @@ -37,8 +37,7 @@ def get(self, name): logger.debug('get: {}:{}'.format(self.bucket, name)) try: return self.client.get_object(self.bucket, str(name)).data - except Exception as e: - print(e) + except minio.error.S3Error: raise errors.MissingExternalFile('Missing s3 key %s' % name) from None def fget(self, name, local_filepath): @@ -60,8 +59,7 @@ def exists(self, name): logger.debug('exists: {}:{}'.format(self.bucket, name)) try: self.client.stat_object(self.bucket, str(name)) - except Exception as e: - print(e) + except minio.error.S3Error: return False return True @@ -69,8 +67,7 @@ def get_size(self, name): logger.debug('get_size: {}:{}'.format(self.bucket, name)) try: return self.client.stat_object(self.bucket, str(name)).size - except Exception as e: - print(e) + except minio.error.S3Error: raise errors.MissingExternalFile from None def remove_object(self, name): From 912a31a0735232bd7062fedf8f70670d43e7222e Mon Sep 17 00:00:00 2001 From: synicix Date: Thu, 10 Dec 2020 14:44:03 -0600 Subject: [PATCH 5/8] Change error handling to be a bit more specific simlar to the old version --- datajoint/s3.py | 23 ++++++++++++++++------- tests/__init__.py | 10 ++++++---- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/datajoint/s3.py b/datajoint/s3.py index 92f044eb8..99f12a6c1 100644 --- a/datajoint/s3.py +++ b/datajoint/s3.py @@ -37,8 +37,11 @@ def get(self, name): logger.debug('get: {}:{}'.format(self.bucket, name)) try: return self.client.get_object(self.bucket, str(name)).data - except minio.error.S3Error: - raise errors.MissingExternalFile('Missing s3 key %s' % name) from None + except minio.error.S3Error as e: + if e.code == 'NoSuchKey': + raise errors.MissingExternalFile('Missing s3 key %s' % name) from None + else: + raise e def fget(self, name, local_filepath): """get file from object name to local filepath""" @@ -59,17 +62,23 @@ def exists(self, name): logger.debug('exists: {}:{}'.format(self.bucket, name)) try: self.client.stat_object(self.bucket, str(name)) - except minio.error.S3Error: - return False + except minio.error.S3Error as e: + if e.code == 'NoSuchKey': + return False + else: + raise e return True def get_size(self, name): logger.debug('get_size: {}:{}'.format(self.bucket, name)) try: return self.client.stat_object(self.bucket, str(name)).size - except minio.error.S3Error: - raise errors.MissingExternalFile from None - + except minio.error.S3Error as e: + if e.code == 'NoSuchKey': + raise errors.MissingExternalFile from None + else: + raise e + def remove_object(self, name): logger.debug('remove_object: {}:{}'.format(self.bucket, name)) try: diff --git a/tests/__init__.py b/tests/__init__.py index a2ecc65e8..6d578f95d 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -136,8 +136,9 @@ def setup_package(): region = "us-east-1" try: minioClient.make_bucket(S3_MIGRATE_BUCKET, location=region) - except minio.error.S3Error: - pass + except minio.error.S3Error as e: + if e.code != 'BucketAlreadyOwnedByYou': + raise e pathlist = Path(source).glob('**/*') for path in pathlist: @@ -149,8 +150,9 @@ def setup_package(): # Add S3 try: minioClient.make_bucket(S3_CONN_INFO['bucket'], location=region) - except minio.error.S3Error: - pass + except minio.error.S3Error as e: + if e.code != 'BucketAlreadyOwnedByYou': + raise e # Add old File Content try: From a11dd92173d71400931fde91172f986db4845985 Mon Sep 17 00:00:00 2001 From: synicix Date: Thu, 10 Dec 2020 14:45:40 -0600 Subject: [PATCH 6/8] Update requirement to set minio>=7.0.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5c84e822c..fa3882448 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,5 +6,5 @@ pandas tqdm networkx pydot -minio +minio>=7.0.0 matplotlib From 0fb4e52504dbf259676742b9f74dfa2c625d9d34 Mon Sep 17 00:00:00 2001 From: synicix Date: Thu, 10 Dec 2020 15:06:08 -0600 Subject: [PATCH 7/8] Remove white space --- datajoint/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datajoint/s3.py b/datajoint/s3.py index 99f12a6c1..d991bcd77 100644 --- a/datajoint/s3.py +++ b/datajoint/s3.py @@ -78,7 +78,7 @@ def get_size(self, name): raise errors.MissingExternalFile from None else: raise e - + def remove_object(self, name): logger.debug('remove_object: {}:{}'.format(self.bucket, name)) try: From b541887789ca9820d752435cc93258ac849acbf0 Mon Sep 17 00:00:00 2001 From: synicix Date: Thu, 10 Dec 2020 15:44:27 -0600 Subject: [PATCH 8/8] Remove additonal white space --- datajoint/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datajoint/s3.py b/datajoint/s3.py index d991bcd77..0e3efb2dd 100644 --- a/datajoint/s3.py +++ b/datajoint/s3.py @@ -78,7 +78,7 @@ def get_size(self, name): raise errors.MissingExternalFile from None else: raise e - + def remove_object(self, name): logger.debug('remove_object: {}:{}'.format(self.bucket, name)) try: