Added initial code for RDS and made S3 more OOP (with Bucket objects)

cloud/amazon/common/aws_util.py

@@ -0,0 +1,7 @@
+from types_extensions import dict_type
+
+
+def extract_aws_response_status_code(resp: dict_type) -> int:
+    if meta_ := resp.get('ResponseMetadata'):
+        return meta_.get('HTTPStatusCode')
+    return 404

cloud/amazon/common/base_service.py

@@ -11,20 +11,30 @@
 
 from cloud.amazon.common.exception_handling import ExceptionLevels
 from cloud.amazon.common.service_availability import ServiceAvailability
-from types_extensions import tuple_type
+from types_extensions import tuple_type, void
 
 
 class BaseAmazonService(abc.ABC):
 
     _backend:  boto3.Session = boto3.Session()
     _client: BaseClient
     default_exception_level: int = ExceptionLevels.RAISE
-    region: str = _backend.region_name
+
+    def __init__(self, profile: str = None, region: str = None, default_exception_level: int = None) -> void:
+
+        self._spawn_session(profile, region)
+
+        if default_exception_level:
+            self.default_exception_level = default_exception_level
 
     @abc.abstractmethod
     def check_service_availability(self) -> int:
         raise NotImplementedError
 
+    @property
+    def region(self) -> str:
+        return self._backend.region_name
+
     @property
     @abc.abstractmethod
     def name(self) -> str:
@@ -46,6 +56,15 @@ def is_named(self) -> bool:
             hasattr(self, 'suffix')
         )
 
+    def _spawn_session(self, profile_name: str, region_name: str) -> void:
+        session_kwargs = {}
+        if profile_name:
+            session_kwargs['profile_name'] = profile_name
+        if region_name:
+            session_kwargs['region_name'] = region_name
+        if len(session_kwargs) > 0:
+            self._backend = boto3.Session(**session_kwargs)
+
     def is_connected(self) -> bool:
         return (self.check_service_availability() & ServiceAvailability.CONNECTED) == ServiceAvailability.CONNECTED
 

cloud/amazon/common/base_service_generated_instance.py

@@ -5,5 +5,19 @@
 
 class BaseSGI(metaclass=abc.ABCMeta):
 
-    def __init__(self, *, exception_level: int) -> void:
+    def __init__(self, *, parent, exception_level: int) -> void:
         self.exception_level: int = exception_level
+        self.parent = parent
+
+    @classmethod
+    @abc.abstractmethod
+    def from_aws_response(cls, aws_resp: dict, parent, exception_level: int, **kwargs) -> 'BaseSGI':
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def create(self, *args, **kwargs) -> 'BaseSGI':
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def delete(self, *args, **kwargs) -> void:
+        raise NotImplementedError

cloud/amazon/common/exception_handling.py

@@ -23,5 +23,27 @@ def __str__(self) -> str:
         return msg
 
 
-class BucketNotEmptyException(Exception):
-    ...
+class MissingParametersException(Exception):
+
+    def __init__(self, msg: str = None, parameter_names: list_type[str] = None) -> void:
+        self.msg = msg or 'Some parameters were missing.'
+        self.parameter_names = parameter_names or ['not given']
+
+    def __str__(self) -> str:
+        msg = f"{self.msg}\nMissing params: {', '.join(self.parameter_names)}"
+        return msg
+
+
+class InvalidAWSResponseException(Exception):
+
+    def __init__(self, expected_dict_structures: list_type[list_type[str]]) -> void:
+        self.expected_dict_structures: list_type[str] = [
+            f"aws_response[{']['.join([path_ for path_ in structure])}]"
+            for structure in expected_dict_structures
+        ]
+
+    def __str__(self) -> str:
+        newl_ = "\n"
+        msg = f"The AWS Response you provided is invalid. The given dict requires the following paths:\n" \
+              f"{newl_.join(self.expected_dict_structures)}"
+        return msg

cloud/amazon/services/__init__.py

@@ -1,2 +1,3 @@
 from .s3 import *
 from .ec2 import *
+from .rds import *

cloud/amazon/services/ec2/__init__.py

@@ -1 +1 @@
-from .ec2 import AmazonEC2
+from ._ec2 import AmazonEC2

cloud/amazon/services/ec2/ec2.py → cloud/amazon/services/ec2/_ec2.py

@@ -12,13 +12,13 @@ class AmazonEC2(BaseAmazonService):
 
     def __init__(self, profile: str = None, region: str = None, default_exception_level: int = None,
                  delimiter: str = '', instance_name_prefix: str = '', instance_name_suffix: str = '') -> void:
-        if profile:
-            self._backend: boto3.Session = boto3.Session(profile_name=profile)
+
+        super().__init__(profile, region, default_exception_level)
+
         self.prefix: str = instance_name_prefix
         self.suffix: str = instance_name_suffix
         self.delimiter: str = delimiter
-        self.default_exception_level: int = default_exception_level or ExceptionLevels.RAISE
-        self.region: str = region or self._backend.region_name
+
         self._client: const(BaseClient) = self._backend.client(AWSServiceNameMapping.EC2, region_name=region)
 
     def check_service_availability(self) -> int:

cloud/amazon/services/rds/__init__.py

@@ -0,0 +1 @@
+from ._rds import AmazonRDS

cloud/amazon/services/rds/_rds.py

@@ -0,0 +1,59 @@
+import warnings
+
+from botocore.client import BaseClient
+
+from cloud.amazon.common.aws_service_name_mapping import AWSServiceNameMapping
+from cloud.amazon.common.base_service import BaseAmazonService
+from cloud.amazon.common.exception_handling import ExceptionLevels
+from cloud.amazon.common.service_availability import ServiceAvailability
+from types_extensions import void, const, list_type, dict_type
+
+
+class AmazonRDS(BaseAmazonService):
+    # plan: List instances, describe one instance, bring up, enable/disable, destroy, backup/replicate
+
+    def __init__(self, profile: str = None, region: str = None, default_exception_level: int = None) -> void:
+
+        super().__init__(profile, region, default_exception_level)
+        self._client: const(BaseClient) = self._backend.client(AWSServiceNameMapping.RDS, region_name=self.region)
+
+    def check_service_availability(self) -> int:
+        # Not done, for now assumed RDS is up.
+        status = ServiceAvailability.OFFLINE
+        if 1 == 1:
+            status |= ServiceAvailability.ONLINE
+        if 2 == 2:
+            status |= ServiceAvailability.CONNECTED
+        return status
+
+    @property
+    def name(self) -> str:
+        return AWSServiceNameMapping.RDS
+
+    def list_db_instances(self, exception_level: int = None) -> list_type[str]:
+        if not self._assert_connection(exception_level):
+            return []
+        try:
+            aws_resp = self._client.describe_db_instances()
+            return [instance_['DBInstanceIdentifier'] for instance_ in aws_resp['DBInstances']]
+        except Exception as e:
+            if exception_level == ExceptionLevels.RAISE:
+                raise
+            if exception_level == ExceptionLevels.WARN:
+                warnings.warn(f"An unknown exception occurred while trying to list databases:\nThe error is:\n{e}")
+
+    def describe_db_instance(self, instance_name: str, exception_level: int = None) -> dict_type:
+        if not self._assert_connection(exception_level):
+            return {}
+        try:
+            aws_resp = self._client.describe_db_instances(
+                DBInstanceIdentifier=instance_name
+            )
+            if instances := aws_resp.get('DBInstances'):
+                return instances[0]
+        except Exception as e:
+            if exception_level == ExceptionLevels.RAISE:
+                raise
+            if exception_level == ExceptionLevels.WARN:
+                warnings.warn(f"An unknown exception occurred while trying to:\n"
+                              f"Describe: {instance_name}\nThe error is:\n{e}")

cloud/amazon/services/rds/_rds_engine.py

@@ -0,0 +1,20 @@
+from meta.config_meta import FinalConfigMeta
+from types_extensions import const
+
+
+class RDSEngine(metaclass=FinalConfigMeta):
+
+    AURORA: const(str) = 'aurora'
+    AURORA_MYSQL: const(str) = 'aurora-mysql'
+    AURORA_POSTGRESQL: const(str) = 'aurora-postgresql'
+    MARIADB: const(str) = 'mariadb'
+    MYSQL: const(str) = 'mysql'
+    ORACLE_EE: const(str) = 'oracle-ee'
+    ORACLE_EE_CDB: const(str) = 'oracle-ee-cdb'
+    ORACLE_SE2: const(str) = 'oracle-se2'
+    ORACLE_SE2_CDB: const(str) = 'oracle-se2-cdb'
+    POSTGRES: const(str) = 'postgres'
+    SQLSERVER_EE: const(str) = 'sqlserver-ee'
+    SQLSERVER_SE: const(str) = 'sqlserver-se'
+    SQLSERVER_EX: const(str) = 'sqlserver-ex'
+    SQLSERVER_WEB: const(str) = 'sqlserver-web'

cloud/amazon/services/rds/_rds_instance.py

@@ -0,0 +1,41 @@
+from typing import Any
+
+from cloud.amazon.common.base_service_generated_instance import BaseSGI
+from properties_and_methods import CachedProperty
+from types_extensions import void, const, list_type, dict_type
+
+
+class AmazonRDSInstance(BaseSGI):
+
+    def __init__(self, db_name: str, db_instance_identifier: str, db_instance_class: str, db_engine: str,
+                 db_port: int, parent, exception_level: int) -> void:
+        super().__init__(parent=parent, exception_level=exception_level)
+        self.db_name: const(str) = db_name
+        self.db_instance_identifier: const(str) = db_instance_identifier
+        self.db_instance_class: const(str) = db_instance_class
+        self.db_engine: const(str) = db_engine
+        self.db_port: const(int) = db_port
+
+    def set_exception_level(self, new_level: int) -> void:
+        self.exception_level = new_level
+        self.invalidate_cached_property_defaults()
+
+    @classmethod
+    def from_aws_response(cls, aws_resp: dict, parent, exception_level: int, **kwargs) -> 'AmazonRDSInstance':
+        ...
+
+    @CachedProperty
+    def defaults(self):
+        return dict(
+            rds_obj=self,
+            exception_level=self.exception_level,
+        )
+
+    def _build_default_params(self, **kwargs) -> dict_type[str, Any]:
+        return {**self.defaults, **kwargs}
+
+    def create(self, acl: str = 'private', region: str = None, **kwargs) -> 'AmazonRDSInstance':
+        ...
+
+    def delete(self, force_delete_contents: bool = False, **kwargs) -> void:
+        ...

cloud/amazon/services/s3/_bucket.py

@@ -1,16 +1,27 @@
 from typing import Any
 
 from cloud.amazon.common.base_service_generated_instance import BaseSGI
+from cloud.amazon.common.exception_handling import InvalidAWSResponseException
 from properties_and_methods import CachedProperty
 from types_extensions import void, const, list_type, dict_type
 
 
 class AmazonS3Bucket(BaseSGI):
 
     def __init__(self, bucket_name: str, parent, exception_level: int) -> void:
-        super().__init__(exception_level=exception_level)
+        super().__init__(parent=parent, exception_level=exception_level)
         self.bucket_name: const(str) = bucket_name
-        self.parent = parent
+
+    @classmethod
+    def from_aws_response(cls, aws_resp: dict, parent, exception_level: int, **kwargs) -> 'AmazonS3Bucket':
+        try:
+            return AmazonS3Bucket(
+                bucket_name=aws_resp['Name'],
+                parent=parent,
+                exception_level=exception_level
+            )
+        except KeyError:
+            raise InvalidAWSResponseException([['Name']])
 
     def set_exception_level(self, new_level: int) -> void:
         self.exception_level = new_level
@@ -19,47 +30,60 @@ def set_exception_level(self, new_level: int) -> void:
     @CachedProperty
     def defaults(self):
         return dict(
-            bucket_name=self.bucket_name,
+            bucket_obj=self,
             apply_format_to_bucket=False,
             exception_level=self.exception_level,
         )
 
-    def _build_default_params(self, kwargs_dict: dict_type[str, Any]) -> dict_type[str, Any]:
-        return {**kwargs_dict, **self.defaults}
+    def _build_default_params(self, **kwargs) -> dict_type[str, Any]:
+        return {**kwargs, **self.defaults}
+
+    def create(self, acl: str = 'private', region: str = None, **kwargs) -> 'AmazonS3Bucket':
+        return self.parent.create_bucket(
+            **self._build_default_params(**kwargs),
+            acl=acl,
+            region=region
+        )
+
+    def delete(self, force_delete_contents: bool = False, **kwargs) -> void:
+        return self.parent.delete_bucket(
+            **self._build_default_params(**kwargs),
+            force_delete_contents=force_delete_contents
+        )
 
     def get_objects(self, mode: str = 'mapping', **kwargs) -> dict:
         return self.parent.get_objects_in_bucket(
-            **self._build_default_params(kwargs),
-            mode=mode,
+            **self._build_default_params(**kwargs),
+            mode=mode
         )
 
     def put_object(self, object_path: str, **kwargs) -> bool:
         return self.parent.put_object_in_bucket(
-            **self._build_default_params(kwargs),
-            object_path=object_path,
+            **self._build_default_params(**kwargs),
+            object_path=object_path
         )
 
     def delete_object(self, object_name: str, **kwargs) -> void:
         return self.parent.delete_object_from_bucket(
-            **self._build_default_params(kwargs),
-            object_name=object_name,
+            **self._build_default_params(**kwargs),
+            object_name=object_name
         )
 
     def delete_objects(self, object_names: list_type[str], **kwargs) -> void:
         return self.parent.delete_objects_from_bucket(
-            **self._build_default_params(kwargs),
-            object_names=object_names,
+            **self._build_default_params(**kwargs),
+            object_names=object_names
         )
 
     def get_all_object_versions(self, object_name: str, **kwargs) -> list_type[dict_type[str, str]]:
         return self.parent.get_all_object_versions(
-            **self._build_default_params(kwargs),
+            **self._build_default_params(**kwargs),
             object_name=object_name
         )
 
     def download_object(self, object_name: str, destination: str, **kwargs) -> void:
         return self.parent.download_object_from_bucket(
-            **self._build_default_params(kwargs),
+            **self._build_default_params(**kwargs),
             object_name=object_name,
             destination=destination
         )