diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index dbad8f2..b7a8946 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7] + python-version: [3.9] steps: - uses: actions/checkout@v2 diff --git a/hasty/__init__.py b/hasty/__init__.py index f279004..7b8e24c 100755 --- a/hasty/__init__.py +++ b/hasty/__init__.py @@ -21,7 +21,7 @@ def int_or_str(value): return value -__version__ = '0.3.9' +__version__ = '0.3.10' VERSION = tuple(map(int_or_str, __version__.split('.'))) __all__ = [ diff --git a/hasty/bucket.py b/hasty/bucket.py new file mode 100644 index 0000000..463325e --- /dev/null +++ b/hasty/bucket.py @@ -0,0 +1,106 @@ +from collections import OrderedDict +from dataclasses import dataclass +from typing import Union, Protocol + +from .constants import BucketProviders +from .hasty_object import HastyObject + +@dataclass +class Credentials(Protocol): + def get_credentials(self): + raise NotImplementedError + + def cloud_provider(self): + raise NotImplementedError + +@dataclass +class DummyCreds(Credentials): + secret: str + + def get_credentials(self): + return {"secret": self.secret, "cloud_provider": BucketProviders.DUMMY} + + def cloud_provider(self): + return BucketProviders.DUMMY + +@dataclass +class GCSCreds(Credentials): + bucket: str + key_json: str + + def get_credentials(self): + return {"bucket_gcs": self.bucket, "key_json": self.key_json, "cloud_provider": BucketProviders.GCS} + + def cloud_provider(self): + return BucketProviders.GCS + +@dataclass +class S3Creds(Credentials): + bucket: str + role: str + + def get_credentials(self): + return {"bucket_s3": self.bucket, "role": self.role, "cloud_provider": BucketProviders.S3} + + def cloud_provider(self): + return BucketProviders.S3 + +@dataclass +class AZCreds(Credentials): + account_name: str + secret_access_key: str + container: str + + def get_credentials(self): + return {"account_name": self.account_name, "secret_access_key": self.secret_access_key, + "container": self.container, "cloud_provider": BucketProviders.AZ} + + def cloud_provider(self): + return BucketProviders.AZ + +class Bucket(HastyObject): + """Class that contains some basic requests and features for bucket management""" + endpoint = '/v1/buckets/{workspace_id}/credentials' + + def __repr__(self): + return self.get__repr__(OrderedDict({"id": self._id, "name": self._name, "cloud_provider": self._cloud_provider})) + + @property + def id(self): + """ + :type: string + """ + return self._id + + @property + def name(self): + """ + :type: string + """ + return self._name + + @property + def cloud_provider(self): + """ + :type: string + """ + return self._cloud_provider + + def _init_properties(self): + self._id = None + self._name = None + self._cloud_provider = None + + def _set_prop_values(self, data): + if "credential_id" in data: + self._id = data["credential_id"] + if "description" in data: + self._name = data["description"] + if "cloud_provider" in data: + self._cloud_provider = data["cloud_provider"] + + @staticmethod + def _create_bucket(requester, workspace_id, name, credentials: Union[DummyCreds, GCSCreds, S3Creds, AZCreds]): + json = {"description": name, "cloud_provider": credentials.cloud_provider(), **credentials.get_credentials()} + data = requester.post(Bucket.endpoint.format(workspace_id=workspace_id), json_data=json) + return Bucket(requester, data) diff --git a/hasty/constants.py b/hasty/constants.py index db5b23a..61b8175 100644 --- a/hasty/constants.py +++ b/hasty/constants.py @@ -41,6 +41,13 @@ class SemanticOrder: CLASS_ORDER = "class_order" +class BucketProviders: + GCS = "gcs" + S3 = "s3" + AZ = "az" + DUMMY = "dummy" + + WAIT_INTERVAL_SEC = 10 VALID_STATUSES = [ImageStatus.New, ImageStatus.Done, ImageStatus.Skipped, ImageStatus.InProgress, ImageStatus.ToReview, diff --git a/hasty/image.py b/hasty/image.py index 66be49e..79cc086 100644 --- a/hasty/image.py +++ b/hasty/image.py @@ -156,6 +156,18 @@ def _upload_from_url(requester, project_id, dataset_id, filename, url, copy_orig return Image(requester, res, {"project_id": project_id, "dataset_id": dataset_id}) + @staticmethod + def _upload_from_bucket(requester, project_id, dataset_id, filename, path, bucket_id, copy_original=False, + external_id: Optional[str] = None): + res = requester.post(Image.endpoint.format(project_id=project_id), + json_data={"dataset_id": dataset_id, + "url": path, + "filename": filename, + "bucket_id": bucket_id, + "copy_original": copy_original, + "external_id": external_id}) + return Image(requester, res, {"project_id": project_id, + "dataset_id": dataset_id}) def get_labels(self): """ Returns image labels (list of `~hasty.Label` objects) diff --git a/hasty/project.py b/hasty/project.py index 2e347bd..b599021 100644 --- a/hasty/project.py +++ b/hasty/project.py @@ -215,6 +215,25 @@ def upload_from_url(self, dataset: Union[Dataset, str], filename: str, url: str, return Image._upload_from_url(self._requester, self._id, dataset_id, filename, url, copy_original=copy_original, external_id=external_id) + def upload_from_bucket(self, dataset: Union[Dataset, str], filename: str, path: str, bucket_id: str, copy_original: Optional[bool] = False, + external_id: Optional[str] = None): + """ + Uploads image from the given bucket + + Args: + dataset (`~hasty.Dataset`, str): Dataset object or id that the image should belongs to + filename (str): Filename of the image + path (str): Path in the bucket + bucket_id (str): Bucket ID (format: UUID) + copy_original (str): If True Hasty makes a copy of the image. Default False. + external_id (str): External ID (optional) + """ + dataset_id = dataset + if isinstance(dataset, Dataset): + dataset_id = dataset.id + return Image._upload_from_bucket(self._requester, self._id, dataset_id, filename, path, bucket_id, copy_original=copy_original, + external_id=external_id) + def get_label_classes(self): """ Get label classes, list of :py:class:`~hasty.LabelClass` objects. diff --git a/hasty/workspace.py b/hasty/workspace.py index c8be312..a98bf02 100644 --- a/hasty/workspace.py +++ b/hasty/workspace.py @@ -1,6 +1,9 @@ +from typing import Union from collections import OrderedDict + from .hasty_object import HastyObject +from .bucket import DummyCreds, GCSCreds, S3Creds, AZCreds, Bucket class Workspace(HastyObject): @@ -34,3 +37,13 @@ def _set_prop_values(self, data): self._id = data["id"] if "name" in data: self._name = data["name"] + + def create_bucket(self, name: str, credentials: Union[DummyCreds, GCSCreds, S3Creds, AZCreds]): + """ + Create a new bucket in the workspace. + + Args: + name (str): Name of the bucket. + credentials (Credentials): Credentials object. + """ + return Bucket._create_bucket(self._requester, self._id, name, credentials) diff --git a/setup.py b/setup.py index a5c3d0f..43d80eb 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ "numpy>=1.16", ], install_requires=["numpy>=1.16", 'requests >= 2.23.0', 'retrying==1.3.3'], - python_requires=">=3.6", + python_requires=">=3.9", classifiers=[ "Development Status :: 4 - Beta", "License :: OSI Approved :: MIT License", diff --git a/tests/test_bucket.py b/tests/test_bucket.py new file mode 100644 index 0000000..6e36693 --- /dev/null +++ b/tests/test_bucket.py @@ -0,0 +1,40 @@ +import unittest + +from tests.utils import get_client + +from hasty.bucket import S3Creds + + +class TestBucketManagement(unittest.TestCase): + def setUp(self): + self.h = get_client() + self.workspace = self.h.get_workspaces()[0] + self.project = self.h.create_project(self.workspace, "Test Project 1") + + def tearDown(self): + self.project.delete() + + def test_bucket_creation(self): + ws = self.h.get_workspaces()[0] + res = ws.create_bucket("test_bucket", S3Creds(bucket="hasty-public-bucket-mounter", role="arn:aws:iam::045521589961:role/hasty-public-bucket-mounter")) + self.assertIsNotNone(res.id) + self.assertEqual("test_bucket", res.name) + self.assertEqual("s3", res.cloud_provider) + + def test_import_image(self): + # create a bucket + bucket = self.workspace.create_bucket("test_bucket", S3Creds(bucket="hasty-public-bucket-mounter", role="arn:aws:iam::045521589961:role/hasty-public-bucket-mounter")) + + # Import an image from the bucket + dataset = self.project.create_dataset("ds2") + img = self.project.upload_from_bucket(dataset, "1645001880-075718046bb2fbf9b8c35d6e88571cd7f91ca1a1.png", + "dummy/1645001880-075718046bb2fbf9b8c35d6e88571cd7f91ca1a1.png", bucket.id) + self.assertEqual("1645001880-075718046bb2fbf9b8c35d6e88571cd7f91ca1a1.png", img.name) + self.assertEqual("ds2", img.dataset_name) + self.assertIsNotNone(img.id) + self.assertEqual(1280, img.width) + self.assertEqual(720, img.height) + + +if __name__ == '__main__': + unittest.main()