From 772d5efc0b706048faf7b3c40fefa7fbda5acb5d Mon Sep 17 00:00:00 2001 From: Xavier Canal Masjuan Date: Tue, 17 Dec 2024 17:28:10 +0100 Subject: [PATCH 01/10] Adding import_from_bucket method in project --- hasty/image.py | 12 ++++++++++++ hasty/project.py | 19 +++++++++++++++++++ tests/test_bucket.py | 10 ++++++++++ 3 files changed, 41 insertions(+) create mode 100644 tests/test_bucket.py diff --git a/hasty/image.py b/hasty/image.py index 66be49e..79cc086 100644 --- a/hasty/image.py +++ b/hasty/image.py @@ -156,6 +156,18 @@ def _upload_from_url(requester, project_id, dataset_id, filename, url, copy_orig return Image(requester, res, {"project_id": project_id, "dataset_id": dataset_id}) + @staticmethod + def _upload_from_bucket(requester, project_id, dataset_id, filename, path, bucket_id, copy_original=False, + external_id: Optional[str] = None): + res = requester.post(Image.endpoint.format(project_id=project_id), + json_data={"dataset_id": dataset_id, + "url": path, + "filename": filename, + "bucket_id": bucket_id, + "copy_original": copy_original, + "external_id": external_id}) + return Image(requester, res, {"project_id": project_id, + "dataset_id": dataset_id}) def get_labels(self): """ Returns image labels (list of `~hasty.Label` objects) diff --git a/hasty/project.py b/hasty/project.py index 2e347bd..a10473e 100644 --- a/hasty/project.py +++ b/hasty/project.py @@ -215,6 +215,25 @@ def upload_from_url(self, dataset: Union[Dataset, str], filename: str, url: str, return Image._upload_from_url(self._requester, self._id, dataset_id, filename, url, copy_original=copy_original, external_id=external_id) + def upload_from_bucket(self, dataset: Union[Dataset, str], filename: str, path: str, bucket_id: str, copy_original: bool = False, + external_id: Optional[str] = None): + """ + Uploads image from the given bucket + + Args: + dataset (`~hasty.Dataset`, str): Dataset object or id that the image should belongs to + filename (str): Filename of the image + path (str): Path in the bucket + bucket_id (str): Bucket ID (format: UUID) + copy_original (str): If True Hasty makes a copy of the image. Default False. + external_id (str): External ID (optional) + """ + dataset_id = dataset + if isinstance(dataset, Dataset): + dataset_id = dataset.id + return Image._upload_from_bucket(self._requester, self._id, dataset_id, filename, path, bucket_id, copy_original=copy_original, + external_id=external_id) + def get_label_classes(self): """ Get label classes, list of :py:class:`~hasty.LabelClass` objects. diff --git a/tests/test_bucket.py b/tests/test_bucket.py new file mode 100644 index 0000000..493b0f1 --- /dev/null +++ b/tests/test_bucket.py @@ -0,0 +1,10 @@ +import unittest + + +class MyTestCase(unittest.TestCase): + def test_something(self): + self.assertEqual(True, False) # add assertion here + + +if __name__ == '__main__': + unittest.main() From 7a27939fc4fb87ec8db6a2afb11ba56d8dfaf523 Mon Sep 17 00:00:00 2001 From: Xavier Canal Masjuan Date: Tue, 17 Dec 2024 18:19:48 +0100 Subject: [PATCH 02/10] Add workspace.create_bucket method --- hasty/bucket.py | 96 ++++++++++++++++++++++++++++++++++++++++++++ hasty/constants.py | 7 ++++ hasty/workspace.py | 13 ++++++ tests/test_bucket.py | 20 +++++++-- 4 files changed, 133 insertions(+), 3 deletions(-) create mode 100644 hasty/bucket.py diff --git a/hasty/bucket.py b/hasty/bucket.py new file mode 100644 index 0000000..d9af837 --- /dev/null +++ b/hasty/bucket.py @@ -0,0 +1,96 @@ +from collections import OrderedDict +from typing import Union + +from constants import BucketProviders +from .hasty_object import HastyObject + +class Credentials: + pass + +class DummyCreds(Credentials): + secret: str + + def get_credentials(self): + return {"secret": self.secret, "cloud_provider": BucketProviders.DUMMY} + + def cloud_provider(self): + return BucketProviders.DUMMY + +class GCSCreds(Credentials): + bucket: str + key_json: str + + def get_credentials(self): + return {"bucket_gcs": self.bucket, "key_json": self.key_json, "cloud_provider": BucketProviders.GCS} + + def cloud_provider(self): + return BucketProviders.GCS + +class S3Creds(Credentials): + bucket: str + role: str + + def get_credentials(self): + return {"bucket_s3": self.bucket, "role": self.role, "cloud_provider": BucketProviders.S3} + + def cloud_provider(self): + return BucketProviders.S3 + +class AZCreds(Credentials): + account_name: str + secret_access_key: str + container: str + + def get_credentials(self): + return {"account_name": self.account_name, "secret_access_key": self.secret_access_key, + "container": self.container, "cloud_provider": BucketProviders.AZ} + + def cloud_provider(self): + return BucketProviders.AZ + +class Bucket(HastyObject): + """Class that contains some basic requests and features for bucket management""" + endpoint = '/v1/buckets/{workspace_id}/credentials' + + def __repr__(self): + return self.get__repr__(OrderedDict({"id": self._id, "name": self._name, "cloud_provider": self._cloud_provider})) + + @property + def id(self): + """ + :type: string + """ + return self._id + + @property + def name(self): + """ + :type: string + """ + return self._name + + @property + def cloud_provider(self): + """ + :type: string + """ + return self._cloud_provider + + def _init_properties(self): + self._id = None + self._name = None + self._cloud_provider = None + + def _set_prop_values(self, data): + if "credential_id" in data: + self._id = data["credential_id"] + if "description" in data: + self._name = data["description"] + if "cloud_provider" in data: + self._cloud_provider = data["cloud_provider"] + + @staticmethod + def _create_bucket(requester, workspace_id, name, credentials: Union[DummyCreds, GCSCreds, S3Creds, AZCreds]): + json = {"description": name, "cloud_provider": credentials.cloud_provider(), **credentials.get_credentials()} + data = requester.post(Bucket.endpoint.format(workspace_id=workspace_id), json_data=json) + return Bucket(requester, data) diff --git a/hasty/constants.py b/hasty/constants.py index db5b23a..61b8175 100644 --- a/hasty/constants.py +++ b/hasty/constants.py @@ -41,6 +41,13 @@ class SemanticOrder: CLASS_ORDER = "class_order" +class BucketProviders: + GCS = "gcs" + S3 = "s3" + AZ = "az" + DUMMY = "dummy" + + WAIT_INTERVAL_SEC = 10 VALID_STATUSES = [ImageStatus.New, ImageStatus.Done, ImageStatus.Skipped, ImageStatus.InProgress, ImageStatus.ToReview, diff --git a/hasty/workspace.py b/hasty/workspace.py index c8be312..a98bf02 100644 --- a/hasty/workspace.py +++ b/hasty/workspace.py @@ -1,6 +1,9 @@ +from typing import Union from collections import OrderedDict + from .hasty_object import HastyObject +from .bucket import DummyCreds, GCSCreds, S3Creds, AZCreds, Bucket class Workspace(HastyObject): @@ -34,3 +37,13 @@ def _set_prop_values(self, data): self._id = data["id"] if "name" in data: self._name = data["name"] + + def create_bucket(self, name: str, credentials: Union[DummyCreds, GCSCreds, S3Creds, AZCreds]): + """ + Create a new bucket in the workspace. + + Args: + name (str): Name of the bucket. + credentials (Credentials): Credentials object. + """ + return Bucket._create_bucket(self._requester, self._id, name, credentials) diff --git a/tests/test_bucket.py b/tests/test_bucket.py index 493b0f1..d181a11 100644 --- a/tests/test_bucket.py +++ b/tests/test_bucket.py @@ -1,9 +1,23 @@ import unittest +from utils import get_client -class MyTestCase(unittest.TestCase): - def test_something(self): - self.assertEqual(True, False) # add assertion here +from hasty.bucket import S3Creds + + +class TestBucketManagement(unittest.TestCase): + def setUp(self): + self.h = get_client() + + def test_bucket_creation(self): + ws = self.h.get_workspaces()[0] + creds = S3Creds() + creds.bucket = "hasty-public-bucket-mounter" + creds.role = "arn:aws:iam::045521589961:role/hasty-public-bucket-mounter" + res = ws.create_bucket("test_bucket", creds) + self.assertIsNotNone(res.id) + self.assertEqual("test_bucket", res.name) + self.assertEqual("s3", res.cloud_provider) if __name__ == '__main__': From 344bb185eabc1033759631f9578192967ed97334 Mon Sep 17 00:00:00 2001 From: Xavier Canal Masjuan Date: Tue, 17 Dec 2024 18:59:27 +0100 Subject: [PATCH 03/10] Adding test to test image upload from bucket --- hasty/project.py | 2 +- tests/test_bucket.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/hasty/project.py b/hasty/project.py index a10473e..b599021 100644 --- a/hasty/project.py +++ b/hasty/project.py @@ -215,7 +215,7 @@ def upload_from_url(self, dataset: Union[Dataset, str], filename: str, url: str, return Image._upload_from_url(self._requester, self._id, dataset_id, filename, url, copy_original=copy_original, external_id=external_id) - def upload_from_bucket(self, dataset: Union[Dataset, str], filename: str, path: str, bucket_id: str, copy_original: bool = False, + def upload_from_bucket(self, dataset: Union[Dataset, str], filename: str, path: str, bucket_id: str, copy_original: Optional[bool] = False, external_id: Optional[str] = None): """ Uploads image from the given bucket diff --git a/tests/test_bucket.py b/tests/test_bucket.py index d181a11..873ddab 100644 --- a/tests/test_bucket.py +++ b/tests/test_bucket.py @@ -8,6 +8,11 @@ class TestBucketManagement(unittest.TestCase): def setUp(self): self.h = get_client() + self.workspace = self.h.get_workspaces()[0] + self.project = self.h.create_project(self.workspace, "Test Project 1") + + def tearDown(self): + self.project.delete() def test_bucket_creation(self): ws = self.h.get_workspaces()[0] @@ -19,6 +24,22 @@ def test_bucket_creation(self): self.assertEqual("test_bucket", res.name) self.assertEqual("s3", res.cloud_provider) + def test_import_image(self): + creds = S3Creds() + creds.bucket = "hasty-public-bucket-mounter" + creds.role = "arn:aws:iam::045521589961:role/hasty-public-bucket-mounter" + bucket = self.workspace.create_bucket("test_bucket", creds) + bucket_id = bucket.id + + # Test import image + dataset = self.project.create_dataset("ds2") + img = self.project.upload_from_bucket(dataset, "1645001880-075718046bb2fbf9b8c35d6e88571cd7f91ca1a1.png", "dummy/1645001880-075718046bb2fbf9b8c35d6e88571cd7f91ca1a1.png", bucket_id) + self.assertEqual("1645001880-075718046bb2fbf9b8c35d6e88571cd7f91ca1a1.png", img.name) + self.assertEqual("ds2", img.dataset_name) + self.assertIsNotNone(img.id) + self.assertEqual(1280, img.width) + self.assertEqual(720, img.height) + if __name__ == '__main__': unittest.main() From 27ffaedd041bc1552829a0d9a28d6a39174f9b05 Mon Sep 17 00:00:00 2001 From: Xavier Canal Masjuan Date: Wed, 18 Dec 2024 12:59:06 +0100 Subject: [PATCH 04/10] bad import for constants --- hasty/bucket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hasty/bucket.py b/hasty/bucket.py index d9af837..bd13a19 100644 --- a/hasty/bucket.py +++ b/hasty/bucket.py @@ -1,7 +1,7 @@ from collections import OrderedDict from typing import Union -from constants import BucketProviders +from .constants import BucketProviders from .hasty_object import HastyObject class Credentials: From 3aeb7105b5a78f3235a2f01f06d55f049120f3ae Mon Sep 17 00:00:00 2001 From: Xavier Canal Masjuan Date: Wed, 18 Dec 2024 12:59:22 +0100 Subject: [PATCH 05/10] bad import for utils --- tests/test_bucket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_bucket.py b/tests/test_bucket.py index 873ddab..8423e3b 100644 --- a/tests/test_bucket.py +++ b/tests/test_bucket.py @@ -1,6 +1,6 @@ import unittest -from utils import get_client +from tests.utils import get_client from hasty.bucket import S3Creds From 84673656a301c184abae14f1635ec6fcba0ed0bf Mon Sep 17 00:00:00 2001 From: Xavier Canal Masjuan Date: Wed, 18 Dec 2024 16:03:41 +0100 Subject: [PATCH 06/10] Using dataclasses for each credentials --- hasty/bucket.py | 16 +++++++++++++--- tests/test_bucket.py | 12 +++++------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/hasty/bucket.py b/hasty/bucket.py index bd13a19..463325e 100644 --- a/hasty/bucket.py +++ b/hasty/bucket.py @@ -1,12 +1,19 @@ from collections import OrderedDict -from typing import Union +from dataclasses import dataclass +from typing import Union, Protocol from .constants import BucketProviders from .hasty_object import HastyObject -class Credentials: - pass +@dataclass +class Credentials(Protocol): + def get_credentials(self): + raise NotImplementedError + + def cloud_provider(self): + raise NotImplementedError +@dataclass class DummyCreds(Credentials): secret: str @@ -16,6 +23,7 @@ def get_credentials(self): def cloud_provider(self): return BucketProviders.DUMMY +@dataclass class GCSCreds(Credentials): bucket: str key_json: str @@ -26,6 +34,7 @@ def get_credentials(self): def cloud_provider(self): return BucketProviders.GCS +@dataclass class S3Creds(Credentials): bucket: str role: str @@ -36,6 +45,7 @@ def get_credentials(self): def cloud_provider(self): return BucketProviders.S3 +@dataclass class AZCreds(Credentials): account_name: str secret_access_key: str diff --git a/tests/test_bucket.py b/tests/test_bucket.py index 8423e3b..584a7a1 100644 --- a/tests/test_bucket.py +++ b/tests/test_bucket.py @@ -25,15 +25,13 @@ def test_bucket_creation(self): self.assertEqual("s3", res.cloud_provider) def test_import_image(self): - creds = S3Creds() - creds.bucket = "hasty-public-bucket-mounter" - creds.role = "arn:aws:iam::045521589961:role/hasty-public-bucket-mounter" - bucket = self.workspace.create_bucket("test_bucket", creds) - bucket_id = bucket.id + # create a bucket + bucket = self.workspace.create_bucket("test_bucket", S3Creds(bucket="hasty-public-bucket-mounter", role="arn:aws:iam::045521589961:role/hasty-public-bucket-mounter")) - # Test import image + # Import an image from the bucket dataset = self.project.create_dataset("ds2") - img = self.project.upload_from_bucket(dataset, "1645001880-075718046bb2fbf9b8c35d6e88571cd7f91ca1a1.png", "dummy/1645001880-075718046bb2fbf9b8c35d6e88571cd7f91ca1a1.png", bucket_id) + img = self.project.upload_from_bucket(dataset, "1645001880-075718046bb2fbf9b8c35d6e88571cd7f91ca1a1.png", + "dummy/1645001880-075718046bb2fbf9b8c35d6e88571cd7f91ca1a1.png", bucket.id) self.assertEqual("1645001880-075718046bb2fbf9b8c35d6e88571cd7f91ca1a1.png", img.name) self.assertEqual("ds2", img.dataset_name) self.assertIsNotNone(img.id) From 72fec204ba130545ce3d130ac6c15bd08532b2be Mon Sep 17 00:00:00 2001 From: Xavier Canal Masjuan Date: Wed, 18 Dec 2024 16:22:04 +0100 Subject: [PATCH 07/10] Using dataclasses for each credentials --- tests/test_bucket.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/test_bucket.py b/tests/test_bucket.py index 584a7a1..6e36693 100644 --- a/tests/test_bucket.py +++ b/tests/test_bucket.py @@ -16,10 +16,7 @@ def tearDown(self): def test_bucket_creation(self): ws = self.h.get_workspaces()[0] - creds = S3Creds() - creds.bucket = "hasty-public-bucket-mounter" - creds.role = "arn:aws:iam::045521589961:role/hasty-public-bucket-mounter" - res = ws.create_bucket("test_bucket", creds) + res = ws.create_bucket("test_bucket", S3Creds(bucket="hasty-public-bucket-mounter", role="arn:aws:iam::045521589961:role/hasty-public-bucket-mounter")) self.assertIsNotNone(res.id) self.assertEqual("test_bucket", res.name) self.assertEqual("s3", res.cloud_provider) From f09b3a37df8ce4d9270bad5f940bad6f8a427756 Mon Sep 17 00:00:00 2001 From: Xavier Canal Masjuan Date: Wed, 18 Dec 2024 16:28:59 +0100 Subject: [PATCH 08/10] Bumping python to 3.9 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a5c3d0f..43d80eb 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ "numpy>=1.16", ], install_requires=["numpy>=1.16", 'requests >= 2.23.0', 'retrying==1.3.3'], - python_requires=">=3.6", + python_requires=">=3.9", classifiers=[ "Development Status :: 4 - Beta", "License :: OSI Approved :: MIT License", From 480be1235bafa083bd4d4e80f10733c99d0f4161 Mon Sep 17 00:00:00 2001 From: Xavier Canal Masjuan Date: Wed, 18 Dec 2024 16:39:02 +0100 Subject: [PATCH 09/10] Bumping python to 3.9 --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index dbad8f2..b7a8946 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7] + python-version: [3.9] steps: - uses: actions/checkout@v2 From 72466c367546730788679e9a8f770bcff7a2cdaa Mon Sep 17 00:00:00 2001 From: Xavier Canal Masjuan Date: Wed, 18 Dec 2024 16:47:14 +0100 Subject: [PATCH 10/10] release new version --- hasty/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hasty/__init__.py b/hasty/__init__.py index f279004..7b8e24c 100755 --- a/hasty/__init__.py +++ b/hasty/__init__.py @@ -21,7 +21,7 @@ def int_or_str(value): return value -__version__ = '0.3.9' +__version__ = '0.3.10' VERSION = tuple(map(int_or_str, __version__.split('.'))) __all__ = [