diff --git a/CHANGELOG.md b/CHANGELOG.md index 8347c2b25..8aa644cac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,8 @@ The types of changes are: * Data seeding for Datadog access tests [#1269](https://github.com/ethyca/fidesops/pull/1269) * Added support for one-to-many relationships for param_values in SaaS configs [#1253](https://github.com/ethyca/fidesops/pull/1253) * Added erasure endpoints for Shopify connector [#1289](https://github.com/ethyca/fidesops/pull/1289) +* Adds ability to send email notification upon privacy request completion [#1282](https://github.com/ethyca/fidesops/pull/1282) + ### Docs * Fix analytics opt out environment variable name [#1170](https://github.com/ethyca/fidesops/pull/1170) diff --git a/data/config/fidesops.toml b/data/config/fidesops.toml index 49a64bb02..6dd94aee2 100644 --- a/data/config/fidesops.toml +++ b/data/config/fidesops.toml @@ -38,3 +38,6 @@ analytics_id = "internal" [admin_ui] enabled = true + +[notifications] +send_request_completion_notification = true diff --git a/docs/fidesops/docs/guides/configuration_reference.md b/docs/fidesops/docs/guides/configuration_reference.md index fa1dcb659..d4287860d 100644 --- a/docs/fidesops/docs/guides/configuration_reference.md +++ b/docs/fidesops/docs/guides/configuration_reference.md @@ -47,6 +47,7 @@ The `fidesops.toml` file should specify the following variables: | `root_username` | `FIDESOPS__SECURITY__ROOT_USERNAME` | string | root_user | None | If set this can be used in conjunction with `root_password` to log in as a root user without first needing to create a user in the database. | | `root_password` | `FIDESOPS__SECURITY__ROOT_PASSWORD` | string | apassword | None | If set this can be used in conjunction with `root_username` to log in as a root user without first needing to create a user in the database. | | `root_user_scopes` | `FIDESOPS__SECURITY__ROOT_USER_SCOPES` | list of strings | ["client:create", "client:update"] | All available scopes | The scopes granted to the root user when logging in with `root_username` and `root_password`. | +| `subject_request_download_link_ttl_seconds` | `FIDESOPS__SECURITY__SUBJECT_REQUEST_DOWNLOAD_LINK_TTL_SECONDS` | int | 86400 | 86400 | Time in seconds for a subject data package download link to remain valid, default to 1 day. | | Execution Variables |---|---|---|---|---| |`privacy_request_delay_timeout` | `FIDESOPS__EXECUTION__PRIVACY_REQUEST_DELAY_TIMEOUT` | int | 3600 | 3600 | The amount of time to wait for actions delaying privacy requests, for example pre and post processing webhooks. |`task_retry_count` | `FIDESOPS__EXECUTION__TASK_RETRY_COUNT` | int | 5 | 0 | The number of times a failed request will be retried @@ -61,6 +62,9 @@ The `fidesops.toml` file should specify the following variables: |`analytics_opt_out` | `FIDESOPS__ROOT_USER__ANALYTICS_OPT_OUT` | bool | False | False | Opt out of sending anonymous usage data to Ethyca to improve the product experience. | Admin UI Variables|---|---|---|---|---| |`enabled` | `FIDESOPS__ADMIN_UI__ENABLED` | bool | False | True | Toggle whether the Admin UI is served from `/` +| Fidesops Notification Variables|---|---|---|---|---| +|`send_request_completion_notification` | `FIDESOPS__NOTIFICATIONS__SEND_REQUEST_COMPLETION_NOTIFICATION` | bool | True | True | Whether a notification will be sent to data subjects upon privacy request completion + ### An example `fidesops.toml` configuration file @@ -93,6 +97,7 @@ oauth_root_client_secret = "fidesopsadminsecret" log_level = "INFO" root_username = "root_user" root_password = "Testpassword1!" +subject_request_download_link_ttl_seconds = 86400 [execution] masking_strict = true @@ -110,6 +115,9 @@ analytics_opt_out = false [admin_ui] enabled = true + +[notifications] +send_request_completion_notification = true ``` Note: The configuration is case-sensitive, so the variables must be specified in `lowercase`. diff --git a/docs/fidesops/docs/guides/email_communications.md b/docs/fidesops/docs/guides/email_communications.md index 4b82444e6..2a05da4e3 100644 --- a/docs/fidesops/docs/guides/email_communications.md +++ b/docs/fidesops/docs/guides/email_communications.md @@ -5,8 +5,9 @@ Fidesops supports configuring third party email servers to handle outbound commu Supported modes of use: -- Subject Identity Verification - sends a verification code to the user's email address prior to processing a subject request. for more information on identity verification, see the [Privacy Requests](privacy_requests.md#subject-identity-verification) guide. +- Subject Identity Verification - sends a verification code to the user's email address prior to processing a subject request. For more information on identity verification, see the [Privacy Requests](privacy_requests.md#subject-identity-verification) guide. - Erasure Request Email Fulfillment - sends an email to configured third parties to process erasures for a given data subject. See [creating email Connectors](#email-third-party-services) for more information. +- Privacy Request Completion Notification - sends an email to user's email address with privacy request completion notification, including a download link to data package, for access requests. For more information on request completion notification, see the [Privacy Requests](privacy_requests.md#request-completion-notification) guide. ## Prerequisites diff --git a/docs/fidesops/docs/guides/privacy_requests.md b/docs/fidesops/docs/guides/privacy_requests.md index ead5b2fc1..6f5fe10de 100644 --- a/docs/fidesops/docs/guides/privacy_requests.md +++ b/docs/fidesops/docs/guides/privacy_requests.md @@ -54,6 +54,17 @@ to continue privacy request execution. Until the Privacy Request identity is ve ``` +## Request Completion Notification + +By default, a request completion email will be sent to users, along with a link to download their data, if applicable. To change this behavior, set the `send_request_completion_notification` +variable in your `fidesops.toml`. You must also set up an [EmailConfig](./email_communications.md) that lets fidesops send automated emails +to your users. If using a custom privacy center, ensure that you intake an email identity, which is required for email notifications throughout fidesops. + +!!! Note +For security purposes, the data package download link is a one-time link and expires in 24 hrs by default. To change TTL, update the `subject_request_download_link_ttl_seconds` +variable in your `fidesops.toml`. + + ## Approve and deny Privacy Requests To review Privacy Requests before they are executed, set the `require_manual_request_approval` variable in your `fidesops.toml` to `TRUE`. diff --git a/fidesops.toml b/fidesops.toml index 275c5dc58..e81c12d72 100644 --- a/fidesops.toml +++ b/fidesops.toml @@ -29,6 +29,7 @@ drp_jwt_secret = "secret" log_level = "INFO" root_username = "root_user" root_password = "Testpassword1!" +subject_request_download_link_ttl_seconds = 86400 [execution] masking_strict = true @@ -44,3 +45,6 @@ analytics_opt_out = false [admin_ui] enabled = true + +[notifications] +send_request_completion_notification = false diff --git a/src/fidesops/ops/common_exceptions.py b/src/fidesops/ops/common_exceptions.py index 3567a85fb..0cf9a2d81 100644 --- a/src/fidesops/ops/common_exceptions.py +++ b/src/fidesops/ops/common_exceptions.py @@ -85,6 +85,10 @@ class StorageConfigNotFoundException(BaseException): """Custom Exception - StorageConfig Not Found""" +class IdentityNotFoundException(BaseException): + """Identity Not Found""" + + class WebhookOrderException(BaseException): """Custom Exception - Issue with webhooks order""" diff --git a/src/fidesops/ops/core/config.py b/src/fidesops/ops/core/config.py index a13febda6..706b5d060 100644 --- a/src/fidesops/ops/core/config.py +++ b/src/fidesops/ops/core/config.py @@ -88,6 +88,7 @@ class FidesopsSecuritySettings(SecuritySettings): log_level: str = "INFO" root_user_scopes: Optional[List[str]] = SCOPE_REGISTRY + subject_request_download_link_ttl_seconds: Optional[int] = 86400 @validator("log_level", pre=True) def validate_log_level(cls, value: str) -> str: @@ -149,6 +150,15 @@ class Config: env_prefix = "FIDESOPS__ADMIN_UI__" +class FidesopsNotificationSettings(FidesSettings): + """Configuration settings for data subject and/or data processor notifications""" + + send_request_completion_notification: Optional[bool] = True + + class Config: + env_prefix = "FIDESOPS__NOTIFICATIONS__" + + class FidesopsConfig(FidesSettings): """Configuration variables for the FastAPI project""" @@ -158,6 +168,7 @@ class FidesopsConfig(FidesSettings): execution: ExecutionSettings root_user: RootUserSettings admin_ui: AdminUiSettings + notifications: FidesopsNotificationSettings port: int is_test_mode: bool = os.getenv("TESTING", "").lower() == "true" @@ -214,6 +225,7 @@ def log_all_config_values(self) -> None: "cors_origins", "encoding", "oauth_access_token_expire_minutes", + "subject_request_download_link_ttl_seconds", ], "execution": [ "task_retry_count", @@ -222,6 +234,7 @@ def log_all_config_values(self) -> None: "require_manual_request_approval", "subject_identity_verification_required", ], + "notifications": ["send_request_completion_notification"], } diff --git a/src/fidesops/ops/email_templates/get_email_template.py b/src/fidesops/ops/email_templates/get_email_template.py index 833dc36f8..2ecf05af2 100644 --- a/src/fidesops/ops/email_templates/get_email_template.py +++ b/src/fidesops/ops/email_templates/get_email_template.py @@ -6,6 +6,8 @@ from fidesops.ops.common_exceptions import EmailTemplateUnhandledActionType from fidesops.ops.email_templates.template_names import ( EMAIL_ERASURE_REQUEST_FULFILLMENT, + PRIVACY_REQUEST_COMPLETE_ACCESS_TEMPLATE, + PRIVACY_REQUEST_COMPLETE_DELETION_TEMPLATE, SUBJECT_IDENTITY_VERIFICATION_TEMPLATE, ) from fidesops.ops.schemas.email.email import EmailActionType @@ -25,6 +27,10 @@ def get_email_template(action_type: EmailActionType) -> Template: return template_env.get_template(SUBJECT_IDENTITY_VERIFICATION_TEMPLATE) if action_type == EmailActionType.EMAIL_ERASURE_REQUEST_FULFILLMENT: return template_env.get_template(EMAIL_ERASURE_REQUEST_FULFILLMENT) + if action_type == EmailActionType.PRIVACY_REQUEST_COMPLETE_ACCESS: + return template_env.get_template(PRIVACY_REQUEST_COMPLETE_ACCESS_TEMPLATE) + if action_type == EmailActionType.PRIVACY_REQUEST_COMPLETE_DELETION: + return template_env.get_template(PRIVACY_REQUEST_COMPLETE_DELETION_TEMPLATE) logger.error("No corresponding template linked to the %s", action_type) raise EmailTemplateUnhandledActionType( diff --git a/src/fidesops/ops/email_templates/template_names.py b/src/fidesops/ops/email_templates/template_names.py index 63e74eace..afcf3cda6 100644 --- a/src/fidesops/ops/email_templates/template_names.py +++ b/src/fidesops/ops/email_templates/template_names.py @@ -1,2 +1,4 @@ SUBJECT_IDENTITY_VERIFICATION_TEMPLATE = "subject_identity_verification.html" EMAIL_ERASURE_REQUEST_FULFILLMENT = "erasure_request_email_fulfillment.html" +PRIVACY_REQUEST_COMPLETE_DELETION_TEMPLATE = "privacy_request_complete_deletion.html" +PRIVACY_REQUEST_COMPLETE_ACCESS_TEMPLATE = "privacy_request_complete_access.html" diff --git a/src/fidesops/ops/email_templates/templates/privacy_request_complete_access.html b/src/fidesops/ops/email_templates/templates/privacy_request_complete_access.html new file mode 100644 index 000000000..72de34102 --- /dev/null +++ b/src/fidesops/ops/email_templates/templates/privacy_request_complete_access.html @@ -0,0 +1,26 @@ + + + + + Privacy Request Complete + + +
+ {% if download_links|length > 1 %} +

+ Your data access has been completed and can be downloaded at the below links. For security purposes, these secret links will expire in 24 hours. +

+ + {% else %} +

+ Your data access has been completed and can be downloaded at {{download_links[0]}}. For security purposes, this secret link will expire in 24 hours. +

+ + {% endif %} +
+ + \ No newline at end of file diff --git a/src/fidesops/ops/email_templates/templates/privacy_request_complete_deletion.html b/src/fidesops/ops/email_templates/templates/privacy_request_complete_deletion.html new file mode 100644 index 000000000..f0232ea4b --- /dev/null +++ b/src/fidesops/ops/email_templates/templates/privacy_request_complete_deletion.html @@ -0,0 +1,14 @@ + + + + + Privacy Request Complete + + +
+

+ Your privacy request for deletion has been completed. +

+
+ + \ No newline at end of file diff --git a/src/fidesops/ops/schemas/email/email.py b/src/fidesops/ops/schemas/email/email.py index f77742acb..05ad08e09 100644 --- a/src/fidesops/ops/schemas/email/email.py +++ b/src/fidesops/ops/schemas/email/email.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Any, Dict, Optional, Union +from typing import Any, Dict, List, Optional, Union from pydantic import BaseModel, Extra @@ -20,6 +20,8 @@ class EmailActionType(Enum): # verify email upon acct creation SUBJECT_IDENTITY_VERIFICATION = "subject_identity_verification" EMAIL_ERASURE_REQUEST_FULFILLMENT = "email_erasure_fulfillment" + PRIVACY_REQUEST_COMPLETE_ACCESS = "privacy_request_complete_access" + PRIVACY_REQUEST_COMPLETE_DELETION = "privacy_request_complete_deletion" class EmailTemplateBodyParams(Enum): @@ -41,6 +43,12 @@ def get_verification_code_ttl_minutes(self) -> int: return self.verification_code_ttl_seconds // 60 +class AccessRequestCompleteBodyParams(BaseModel): + """Body params required for privacy request completion access email template""" + + download_links: List[str] + + class EmailForActionType(BaseModel): """Email details that depend on action type""" diff --git a/src/fidesops/ops/service/email/email_dispatch_service.py b/src/fidesops/ops/service/email/email_dispatch_service.py index d2b44235a..2784897b1 100644 --- a/src/fidesops/ops/service/email/email_dispatch_service.py +++ b/src/fidesops/ops/service/email/email_dispatch_service.py @@ -1,5 +1,5 @@ import logging -from typing import Any, Optional, Union +from typing import Any, Optional import requests from requests import Response @@ -8,14 +8,12 @@ from fidesops.ops.common_exceptions import EmailDispatchException from fidesops.ops.email_templates import get_email_template from fidesops.ops.models.email import EmailConfig -from fidesops.ops.models.privacy_request import EmailRequestFulfillmentBodyParams from fidesops.ops.schemas.email.email import ( EmailActionType, EmailForActionType, EmailServiceDetails, EmailServiceSecrets, EmailServiceType, - SubjectIdentityVerificationBodyParams, ) from fidesops.ops.util.logger import Pii @@ -26,10 +24,7 @@ def dispatch_email( db: Session, action_type: EmailActionType, to_email: Optional[str], - email_body_params: Union[ - SubjectIdentityVerificationBodyParams, - EmailRequestFulfillmentBodyParams, - ], + email_body_params: Any, ) -> None: if not to_email: raise EmailDispatchException("No email supplied.") @@ -82,6 +77,22 @@ def _build_email( {"dataset_collection_action_required": body_params} ), ) + if action_type == EmailActionType.PRIVACY_REQUEST_COMPLETE_ACCESS: + base_template = get_email_template(action_type) + return EmailForActionType( + subject="Your data is ready to be downloaded", + body=base_template.render( + { + "download_links": body_params.download_links, + } + ), + ) + if action_type == EmailActionType.PRIVACY_REQUEST_COMPLETE_DELETION: + base_template = get_email_template(action_type) + return EmailForActionType( + subject="Your data has been deleted", + body=base_template.render(), + ) logger.error("Email action type %s is not implemented", action_type) raise EmailDispatchException(f"Email action type {action_type} is not implemented") diff --git a/src/fidesops/ops/service/privacy_request/request_runner_service.py b/src/fidesops/ops/service/privacy_request/request_runner_service.py index b7cd9ef2d..60adea252 100644 --- a/src/fidesops/ops/service/privacy_request/request_runner_service.py +++ b/src/fidesops/ops/service/privacy_request/request_runner_service.py @@ -1,7 +1,7 @@ import logging import random from datetime import datetime, timedelta -from typing import ContextManager, Dict, List, Optional, Set +from typing import Any, ContextManager, Dict, List, Optional, Set from celery import Task from celery.utils.log import get_task_logger @@ -15,6 +15,7 @@ from fidesops.ops.common_exceptions import ( ClientUnsuccessfulException, EmailDispatchException, + IdentityNotFoundException, PrivacyRequestPaused, ) from fidesops.ops.core.config import config @@ -36,9 +37,15 @@ from fidesops.ops.models.privacy_request import ( PrivacyRequest, PrivacyRequestStatus, + ProvidedIdentityType, can_run_checkpoint, ) +from fidesops.ops.schemas.email.email import ( + AccessRequestCompleteBodyParams, + EmailActionType, +) from fidesops.ops.service.connectors.email_connector import email_connector_erasure_send +from fidesops.ops.service.email.email_dispatch_service import dispatch_email from fidesops.ops.service.storage.storage_uploader_service import upload from fidesops.ops.task.filter_results import filter_data_categories from fidesops.ops.task.graph_task import ( @@ -123,11 +130,11 @@ def upload_access_results( access_result: Dict[str, List[Row]], dataset_graph: DatasetGraph, privacy_request: PrivacyRequest, -) -> None: +) -> List[str]: """Process the data uploads after the access portion of the privacy request has completed""" + download_urls: List[str] = [] if not access_result: logging.info("No results returned for access request %s", privacy_request.id) - for rule in policy.get_rules_for_action(action_type=ActionType.access): if not rule.storage_destination: raise common_exceptions.RuleValidationError( @@ -145,12 +152,14 @@ def upload_access_results( privacy_request.id, ) try: - upload( + download_url: Optional[str] = upload( db=session, request_id=privacy_request.id, data=filtered_results, storage_key=rule.storage_destination.key, # type: ignore ) + if download_url: + download_urls.append(download_url) except common_exceptions.StorageUploadError as exc: logging.error( "Error uploading subject access data for rule %s on policy %s and privacy request %s : %s", @@ -160,6 +169,7 @@ def upload_access_results( Pii(str(exc)), ) privacy_request.status = PrivacyRequestStatus.error + return download_urls def queue_privacy_request( @@ -168,6 +178,7 @@ def queue_privacy_request( from_step: Optional[str] = None, ) -> str: cache: FidesopsRedis = get_cache() + logger.info("queueing privacy request") task = run_privacy_request.delay( privacy_request_id=privacy_request_id, from_webhook_id=from_webhook_id, @@ -207,7 +218,7 @@ async def run_privacy_request( from_webhook_id: Optional[str] = None, from_step: Optional[str] = None, ) -> None: - # pylint: disable=too-many-locals, too-many-statements + # pylint: disable=too-many-locals, too-many-statements, too-many-return-statements, too-many-branches """ Dispatch a privacy_request into the execution layer by: 1. Generate a graph from all the currently configured datasets @@ -260,6 +271,7 @@ async def run_privacy_request( dataset_graph = DatasetGraph(*dataset_graphs) identity_data = privacy_request.get_cached_identity_data() connection_configs = ConnectionConfig.all(db=session) + access_result_urls: List[str] = [] if can_run_checkpoint( request_checkpoint=CurrentStep.access, from_checkpoint=resume_step @@ -272,8 +284,7 @@ async def run_privacy_request( identity=identity_data, session=session, ) - - upload_access_results( + access_result_urls = upload_access_results( session, policy, access_result, @@ -343,7 +354,19 @@ async def run_privacy_request( ) if not proceed: return - + if config.notifications.send_request_completion_notification: + try: + initiate_privacy_request_completion_email( + session, policy, access_result_urls, identity_data + ) + except (IdentityNotFoundException, EmailDispatchException) as e: + privacy_request.error_processing(db=session) + # If dev mode, log traceback + await fideslog_graph_failure( + failed_graph_analytics_event(privacy_request, e) + ) + _log_exception(e, config.dev_mode) + return privacy_request.finished_processing_at = datetime.utcnow() AuditLog.create( db=session, @@ -355,8 +378,42 @@ async def run_privacy_request( }, ) privacy_request.status = PrivacyRequestStatus.complete - privacy_request.save(db=session) logging.info("Privacy request %s run completed.", privacy_request.id) + privacy_request.save(db=session) + + +def initiate_privacy_request_completion_email( + session: Session, + policy: Policy, + access_result_urls: List[str], + identity_data: Dict[str, Any], +) -> None: + """ + :param session: SQLAlchemy Session + :param policy: Policy + :param access_result_urls: list of urls generated by access request upload + :param identity_data: Dict of identity data + """ + if not identity_data.get(ProvidedIdentityType.email.value): + raise IdentityNotFoundException( + "Identity email was not found, so request completion email could not be sent." + ) + if policy.get_rules_for_action(action_type=ActionType.access): + dispatch_email( + db=session, + action_type=EmailActionType.PRIVACY_REQUEST_COMPLETE_ACCESS, + to_email=identity_data.get(ProvidedIdentityType.email.value), + email_body_params=AccessRequestCompleteBodyParams( + download_links=access_result_urls + ), + ) + if policy.get_rules_for_action(action_type=ActionType.erasure): + dispatch_email( + db=session, + action_type=EmailActionType.PRIVACY_REQUEST_COMPLETE_DELETION, + to_email=identity_data.get(ProvidedIdentityType.email.value), + email_body_params=None, + ) def initiate_paused_privacy_request_followup(privacy_request: PrivacyRequest) -> None: diff --git a/src/fidesops/ops/service/storage/storage_authenticator_service.py b/src/fidesops/ops/service/storage/storage_authenticator_service.py index c28a16380..7a4dd7e69 100644 --- a/src/fidesops/ops/service/storage/storage_authenticator_service.py +++ b/src/fidesops/ops/service/storage/storage_authenticator_service.py @@ -1,5 +1,5 @@ import logging -from typing import Any, Dict +from typing import Any from botocore.exceptions import ClientError from requests import RequestException @@ -7,8 +7,8 @@ from fidesops.ops.schemas.storage.storage import ( SUPPORTED_STORAGE_SECRETS, S3AuthMethod, - StorageSecrets, StorageSecretsOnetrust, + StorageSecretsS3, StorageType, ) from fidesops.ops.util.storage_authenticator import ( @@ -28,10 +28,10 @@ def secrets_are_valid( return uploader(secrets) -def _s3_authenticator(secrets: Dict[StorageSecrets, Any]) -> bool: +def _s3_authenticator(secrets: StorageSecretsS3) -> bool: """Authenticates secrets for s3, returns true if secrets are valid""" try: - get_s3_session(S3AuthMethod.SECRET_KEYS, secrets) + get_s3_session(S3AuthMethod.SECRET_KEYS.value, secrets.dict()) # type: ignore return True except ClientError: return False diff --git a/src/fidesops/ops/service/storage/storage_uploader_service.py b/src/fidesops/ops/service/storage/storage_uploader_service.py index 825e9c5ee..84caebb75 100644 --- a/src/fidesops/ops/service/storage/storage_uploader_service.py +++ b/src/fidesops/ops/service/storage/storage_uploader_service.py @@ -21,7 +21,14 @@ def upload( db: Session, *, request_id: str, data: Dict, storage_key: FidesOpsKey ) -> str: - """Retrieves storage configs and calls appropriate upload method""" + """ + Retrieves storage configs and calls appropriate upload method + :param db: SQLAlchemy Session + :param request_id: Request id + :param data: Dict of data to upload + :param storage_key: Key representing where to upload data + :return str representing location of upload (url or simply a description of where to find the data) + """ config: Optional[StorageConfig] = StorageConfig.get_by( db=db, field="key", value=storage_key ) diff --git a/src/fidesops/ops/tasks/storage.py b/src/fidesops/ops/tasks/storage.py index 32c67814b..feeff9528 100644 --- a/src/fidesops/ops/tasks/storage.py +++ b/src/fidesops/ops/tasks/storage.py @@ -11,6 +11,7 @@ import pandas as pd import requests +from boto3 import Session from botocore.exceptions import ClientError, ParamValidationError from fideslib.cryptography.cryptographic_util import bytes_to_b64_str @@ -28,7 +29,6 @@ logger = logging.getLogger(__name__) - LOCAL_FIDES_UPLOAD_DIRECTORY = "fides_uploads" @@ -96,6 +96,27 @@ def write_to_in_memory_buffer( raise NotImplementedError(f"No handling for response format {resp_format}.") +def create_presigned_url_for_s3( + s3_client: Session, bucket_name: str, object_name: str +) -> str: + """ "Generate a presigned URL to share an S3 object + + :param s3_client: s3 base client + :param bucket_name: string + :param object_name: string + :return: Presigned URL as string. + """ + + response = s3_client.generate_presigned_url( + "get_object", + Params={"Bucket": bucket_name, "Key": object_name}, + ExpiresIn=config.security.subject_request_download_link_ttl_seconds, + ) + + # The response contains the presigned URL + return response + + def upload_to_s3( # pylint: disable=R0913 storage_secrets: Dict[StorageSecrets, Any], data: Dict, @@ -109,18 +130,28 @@ def upload_to_s3( # pylint: disable=R0913 logger.info("Starting S3 Upload of %s", file_key) try: my_session = get_s3_session(auth_method, storage_secrets) - - s3 = my_session.client("s3") + s3_client = my_session.client("s3") # handles file chunking - s3.upload_fileobj( - Fileobj=write_to_in_memory_buffer(resp_format, data, request_id), - Bucket=bucket_name, - Key=file_key, + try: + s3_client.upload_fileobj( + Fileobj=write_to_in_memory_buffer(resp_format, data, request_id), + Bucket=bucket_name, + Key=file_key, + ) + except Exception as e: + logger.error("Encountered error while uploading s3 object: %s", e) + raise e + + presigned_url: str = create_presigned_url_for_s3( + s3_client, bucket_name, file_key ) - # todo- move to outbound_urn_registry - return f"https://{bucket_name}.s3.amazonaws.com/{file_key}" + + return presigned_url except ClientError as e: + logger.error( + "Encountered error while uploading and generating link for s3 object: %s", e + ) raise e except ParamValidationError as e: raise ValueError(f"The parameters you provided are incorrect: {e}") @@ -147,7 +178,7 @@ def upload_to_onetrust( data=payload, headers=headers, ) - return "success" + return "onetrust" def _handle_json_encoding(field: Any) -> str: @@ -169,4 +200,4 @@ def upload_to_local(payload: Dict, file_key: str, request_id: str) -> str: with open(filename, "w") as file: # pylint: disable=W1514 file.write(data_str) - return "success" + return "your local fides_uploads folder" diff --git a/src/fidesops/ops/util/storage_authenticator.py b/src/fidesops/ops/util/storage_authenticator.py index d47a12b58..8a77ccb21 100644 --- a/src/fidesops/ops/util/storage_authenticator.py +++ b/src/fidesops/ops/util/storage_authenticator.py @@ -41,8 +41,8 @@ def get_s3_session( logger.info("Successfully created automatic session") return session - logger.error("No S3 session created. Auth method used: %s", auth_method) - raise ValueError(f"No S3 session created. Auth method used: {auth_method}") + logger.error("Auth method not supported for S3: %s", auth_method) + raise ValueError(f"Auth method not supported for S3: {auth_method}") def get_onetrust_access_token(client_id: str, client_secret: str, hostname: str) -> str: diff --git a/tests/ops/api/v1/endpoints/test_config_endpoints.py b/tests/ops/api/v1/endpoints/test_config_endpoints.py index 95286f1c5..e2c83556d 100644 --- a/tests/ops/api/v1/endpoints/test_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_config_endpoints.py @@ -35,6 +35,7 @@ def test_get_config( "cors_origins", "encoding", "oauth_access_token_expire_minutes", + "subject_request_download_link_ttl_seconds", ] ) ) diff --git a/tests/ops/api/v1/endpoints/test_storage_endpoints.py b/tests/ops/api/v1/endpoints/test_storage_endpoints.py index a434b8e12..67390d221 100644 --- a/tests/ops/api/v1/endpoints/test_storage_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_storage_endpoints.py @@ -554,11 +554,11 @@ def test_put_s3_config_secrets_and_verify( response = api_client.put(url, headers=auth_header, json=payload) assert 200 == response.status_code get_s3_session_mock.assert_called_once_with( - S3AuthMethod.SECRET_KEYS, - StorageSecretsS3( - aws_access_key_id=payload["aws_access_key_id"], - aws_secret_access_key=payload["aws_secret_access_key"], - ), + S3AuthMethod.SECRET_KEYS.value, + { + "aws_access_key_id": payload["aws_access_key_id"], + "aws_secret_access_key": payload["aws_secret_access_key"], + }, ) @mock.patch( diff --git a/tests/ops/conftest.py b/tests/ops/conftest.py index 7afe7da4c..439396cee 100644 --- a/tests/ops/conftest.py +++ b/tests/ops/conftest.py @@ -255,6 +255,15 @@ def subject_identity_verification_not_required(): config.execution.subject_identity_verification_required = original_value +@pytest.fixture(autouse=True, scope="function") +def privacy_request_complete_email_notification_disabled(): + """Disable request completion email for most tests unless overridden""" + original_value = config.notifications.send_request_completion_notification + config.notifications.send_request_completion_notification = False + yield + config.notifications.send_request_completion_notification = original_value + + @pytest.fixture(scope="session", autouse=True) def event_loop(): try: diff --git a/tests/ops/fixtures/application_fixtures.py b/tests/ops/fixtures/application_fixtures.py index 8f7a9da10..d18989815 100644 --- a/tests/ops/fixtures/application_fixtures.py +++ b/tests/ops/fixtures/application_fixtures.py @@ -17,6 +17,7 @@ from sqlalchemy.orm.exc import ObjectDeletedError from fidesops.ops.api.v1.scope_registry import PRIVACY_REQUEST_READ, SCOPE_REGISTRY +from fidesops.ops.core.config import config from fidesops.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, @@ -293,6 +294,77 @@ def policy_post_execution_webhooks( pass +@pytest.fixture(scope="function") +def access_and_erasure_policy( + db: Session, + oauth_client: ClientDetail, + storage_config: StorageConfig, +) -> Generator: + access_and_erasure_policy = Policy.create( + db=db, + data={ + "name": "example access and erasure policy", + "key": "example_access_erasure_policy", + "client_id": oauth_client.id, + }, + ) + access_rule = Rule.create( + db=db, + data={ + "action_type": ActionType.access.value, + "client_id": oauth_client.id, + "name": "Access Request Rule", + "policy_id": access_and_erasure_policy.id, + "storage_destination_id": storage_config.id, + }, + ) + access_rule_target = RuleTarget.create( + db=db, + data={ + "client_id": oauth_client.id, + "data_category": DataCategory("user").value, + "rule_id": access_rule.id, + }, + ) + erasure_rule = Rule.create( + db=db, + data={ + "action_type": ActionType.erasure.value, + "client_id": oauth_client.id, + "name": "Erasure Rule", + "policy_id": access_and_erasure_policy.id, + "masking_strategy": { + "strategy": "null_rewrite", + "configuration": {}, + }, + }, + ) + + erasure_rule_target = RuleTarget.create( + db=db, + data={ + "client_id": oauth_client.id, + "data_category": DataCategory("user.name").value, + "rule_id": erasure_rule.id, + }, + ) + yield access_and_erasure_policy + try: + access_rule_target.delete(db) + erasure_rule_target.delete(db) + except ObjectDeletedError: + pass + try: + access_rule.delete(db) + erasure_rule.delete(db) + except ObjectDeletedError: + pass + try: + access_and_erasure_policy.delete(db) + except ObjectDeletedError: + pass + + @pytest.fixture(scope="function") def erasure_policy( db: Session, @@ -811,10 +883,13 @@ def _create_privacy_request_for_policy( db=db, data=data, ) + email_identity = "test@example.com" + identity_kwargs = {"email": email_identity} + pr.cache_identity(identity_kwargs) pr.persist_identity( db=db, identity=PrivacyRequestIdentity( - email="test@example.com", + email=email_identity, phone_number="+1 234 567 8910", ), ) diff --git a/tests/ops/service/privacy_request/request_runner_service_test.py b/tests/ops/service/privacy_request/request_runner_service_test.py index a9a22fba9..ef61c6f3a 100644 --- a/tests/ops/service/privacy_request/request_runner_service_test.py +++ b/tests/ops/service/privacy_request/request_runner_service_test.py @@ -1,7 +1,7 @@ import time from typing import Any, Dict, List, Set from unittest import mock -from unittest.mock import Mock +from unittest.mock import ANY, Mock, call from uuid import uuid4 import pydash @@ -13,6 +13,8 @@ from fidesops.ops.common_exceptions import ( ClientUnsuccessfulException, + EmailDispatchException, + IdentityNotFoundException, PrivacyRequestPaused, ) from fidesops.ops.core.config import config @@ -28,7 +30,11 @@ PrivacyRequest, PrivacyRequestStatus, ) -from fidesops.ops.schemas.email.email import EmailForActionType +from fidesops.ops.schemas.email.email import ( + AccessRequestCompleteBodyParams, + EmailActionType, + EmailForActionType, +) from fidesops.ops.schemas.external_https import SecondPartyResponseFormat from fidesops.ops.schemas.masking.masking_configuration import ( HmacMaskingConfiguration, @@ -56,23 +62,47 @@ PRIVACY_REQUEST_TASK_TIMEOUT_EXTERNAL = 30 +@pytest.fixture(scope="function") +def privacy_request_complete_email_notification_enabled(): + """Enable request completion email""" + original_value = config.notifications.send_request_completion_notification + config.notifications.send_request_completion_notification = True + yield + config.notifications.send_request_completion_notification = original_value + + +@mock.patch( + "fidesops.ops.service.privacy_request.request_runner_service.dispatch_email" +) @mock.patch("fidesops.ops.service.privacy_request.request_runner_service.upload") -def test_policy_upload_called( +def test_policy_upload_dispatch_email_called( upload_mock: Mock, + mock_email_dispatch: Mock, privacy_request_status_pending: PrivacyRequest, run_privacy_request_task, + privacy_request_complete_email_notification_enabled, ) -> None: + upload_mock.return_value = "http://www.data-download-url" run_privacy_request_task.delay(privacy_request_status_pending.id).get( timeout=PRIVACY_REQUEST_TASK_TIMEOUT ) assert upload_mock.called + assert mock_email_dispatch.call_count == 1 +@mock.patch( + "fidesops.ops.service.privacy_request.request_runner_service.dispatch_email" +) +@mock.patch("fidesops.ops.service.privacy_request.request_runner_service.upload") def test_start_processing_sets_started_processing_at( + upload_mock: Mock, + mock_email_dispatch: Mock, db: Session, privacy_request_status_pending: PrivacyRequest, run_privacy_request_task, + privacy_request_complete_email_notification_enabled, ) -> None: + upload_mock.return_value = "http://www.data-download-url" updated_at = privacy_request_status_pending.updated_at assert privacy_request_status_pending.started_processing_at is None run_privacy_request_task.delay(privacy_request_status_pending.id).get( @@ -83,12 +113,22 @@ def test_start_processing_sets_started_processing_at( assert privacy_request_status_pending.started_processing_at is not None assert privacy_request_status_pending.updated_at > updated_at + assert mock_email_dispatch.call_count == 1 + +@mock.patch( + "fidesops.ops.service.privacy_request.request_runner_service.dispatch_email" +) +@mock.patch("fidesops.ops.service.privacy_request.request_runner_service.upload") def test_start_processing_doesnt_overwrite_started_processing_at( + upload_mock: Mock, + mock_email_dispatch: Mock, db: Session, privacy_request: PrivacyRequest, run_privacy_request_task, + privacy_request_complete_email_notification_enabled, ) -> None: + upload_mock.return_value = "http://www.data-download-url" before = privacy_request.started_processing_at assert before is not None updated_at = privacy_request.updated_at @@ -101,6 +141,8 @@ def test_start_processing_doesnt_overwrite_started_processing_at( assert privacy_request.started_processing_at == before assert privacy_request.updated_at > updated_at + assert mock_email_dispatch.call_count == 1 + @mock.patch( "fidesops.ops.service.privacy_request.request_runner_service.upload_access_results" @@ -110,6 +152,7 @@ def test_halts_proceeding_if_cancelled( db: Session, privacy_request_status_canceled: PrivacyRequest, run_privacy_request_task, + privacy_request_complete_email_notification_enabled, ) -> None: assert privacy_request_status_canceled.status == PrivacyRequestStatus.canceled run_privacy_request_task.delay(privacy_request_status_canceled.id).get( @@ -124,6 +167,12 @@ def test_halts_proceeding_if_cancelled( assert not upload_access_results_mock.called +@mock.patch( + "fidesops.ops.service.privacy_request.request_runner_service.dispatch_email" +) +@mock.patch( + "fidesops.ops.service.privacy_request.request_runner_service.upload_access_results" +) @mock.patch( "fidesops.ops.service.privacy_request.request_runner_service.run_webhooks_and_report_status", ) @@ -135,11 +184,15 @@ def test_from_graph_resume_does_not_run_pre_webhooks( run_erasure, run_access, run_webhooks, + upload_mock: Mock, + mock_email_dispatch, db: Session, privacy_request: PrivacyRequest, run_privacy_request_task, erasure_policy, + privacy_request_complete_email_notification_enabled, ) -> None: + upload_mock.return_value = "http://www.data-download-url" privacy_request.started_processing_at = None privacy_request.policy = erasure_policy privacy_request.save(db) @@ -161,7 +214,12 @@ def test_from_graph_resume_does_not_run_pre_webhooks( assert run_access.call_count == 1 # Access request runs assert run_erasure.call_count == 1 # Erasure request runs + assert mock_email_dispatch.call_count == 1 + +@mock.patch( + "fidesops.ops.service.privacy_request.request_runner_service.dispatch_email" +) @mock.patch( "fidesops.ops.service.privacy_request.request_runner_service.run_webhooks_and_report_status", ) @@ -173,10 +231,12 @@ def test_resume_privacy_request_from_erasure( run_erasure, run_access, run_webhooks, + mock_email_dispatch, db: Session, privacy_request: PrivacyRequest, run_privacy_request_task, erasure_policy, + privacy_request_complete_email_notification_enabled, ) -> None: privacy_request.started_processing_at = None privacy_request.policy = erasure_policy @@ -199,6 +259,8 @@ def test_resume_privacy_request_from_erasure( assert run_access.call_count == 0 # Access request skipped assert run_erasure.call_count == 1 # Erasure request runs + assert mock_email_dispatch.call_count == 1 + def get_privacy_request_results( db, @@ -1773,3 +1835,220 @@ def test_email_connector_no_updates_needed( assert ( mailgun_send.called is False ), "Email not sent because no updates are needed. Data category doesn't apply to any of the collections." + + +class TestPrivacyRequestsEmailNotifications: + @pytest.fixture(scope="function") + def privacy_request_complete_email_notification_enabled(self): + """Enable request completion email""" + original_value = config.notifications.send_request_completion_notification + config.notifications.send_request_completion_notification = True + yield + config.notifications.send_request_completion_notification = original_value + + @pytest.mark.integration_postgres + @pytest.mark.integration + @mock.patch( + "fidesops.ops.service.privacy_request.request_runner_service.dispatch_email" + ) + def test_email_complete_send_erasure( + self, + mailgun_send, + postgres_integration_db, + postgres_example_test_dataset_config, + cache, + db, + generate_auth_header, + erasure_policy, + read_connection_config, + email_config, + privacy_request_complete_email_notification_enabled, + run_privacy_request_task, + ): + customer_email = "customer-1@example.com" + data = { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": erasure_policy.key, + "identity": {"email": customer_email}, + } + + pr = get_privacy_request_results( + db, + erasure_policy, + run_privacy_request_task, + data, + ) + pr.delete(db=db) + + mailgun_send.assert_called_once() + + @pytest.mark.integration_postgres + @pytest.mark.integration + @mock.patch( + "fidesops.ops.service.privacy_request.request_runner_service.dispatch_email" + ) + @mock.patch("fidesops.ops.service.privacy_request.request_runner_service.upload") + def test_email_complete_send_access( + self, + upload_mock, + mailgun_send, + postgres_integration_db, + postgres_example_test_dataset_config, + cache, + db, + generate_auth_header, + policy, + read_connection_config, + email_config, + privacy_request_complete_email_notification_enabled, + run_privacy_request_task, + ): + upload_mock.return_value = "http://www.data-download-url" + customer_email = "customer-1@example.com" + data = { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": {"email": customer_email}, + } + + pr = get_privacy_request_results( + db, + policy, + run_privacy_request_task, + data, + ) + pr.delete(db=db) + + mailgun_send.assert_called_once() + + @pytest.mark.integration_postgres + @pytest.mark.integration + @mock.patch( + "fidesops.ops.service.privacy_request.request_runner_service.dispatch_email" + ) + @mock.patch("fidesops.ops.service.privacy_request.request_runner_service.upload") + def test_email_complete_send_access_and_erasure( + self, + upload_mock, + mailgun_send, + postgres_integration_db, + postgres_example_test_dataset_config, + cache, + db, + generate_auth_header, + access_and_erasure_policy, + read_connection_config, + email_config, + privacy_request_complete_email_notification_enabled, + run_privacy_request_task, + ): + upload_mock.return_value = "http://www.data-download-url" + customer_email = "customer-1@example.com" + data = { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": access_and_erasure_policy.key, + "identity": {"email": customer_email}, + } + + pr = get_privacy_request_results( + db, + access_and_erasure_policy, + run_privacy_request_task, + data, + ) + pr.delete(db=db) + + mailgun_send.assert_has_calls( + [ + call( + db=ANY, + action_type=EmailActionType.PRIVACY_REQUEST_COMPLETE_ACCESS, + to_email=customer_email, + email_body_params=AccessRequestCompleteBodyParams( + download_links=[upload_mock.return_value] + ), + ), + call( + db=ANY, + action_type=EmailActionType.PRIVACY_REQUEST_COMPLETE_DELETION, + to_email=customer_email, + email_body_params=None, + ), + ], + any_order=True, + ) + + @pytest.mark.integration_postgres + @pytest.mark.integration + @mock.patch("fidesops.ops.service.email.email_dispatch_service._mailgun_dispatcher") + @mock.patch("fidesops.ops.service.privacy_request.request_runner_service.upload") + def test_email_complete_send_access_no_email_config( + self, + upload_mock, + mailgun_send, + postgres_integration_db, + postgres_example_test_dataset_config, + cache, + db, + generate_auth_header, + policy, + read_connection_config, + privacy_request_complete_email_notification_enabled, + run_privacy_request_task, + ): + upload_mock.return_value = "http://www.data-download-url" + customer_email = "customer-1@example.com" + data = { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": {"email": customer_email}, + } + + pr = get_privacy_request_results( + db, + policy, + run_privacy_request_task, + data, + ) + db.refresh(pr) + assert pr.status == PrivacyRequestStatus.error + pr.delete(db=db) + + assert mailgun_send.called is False + + @pytest.mark.integration_postgres + @pytest.mark.integration + @mock.patch("fidesops.ops.service.email.email_dispatch_service._mailgun_dispatcher") + @mock.patch("fidesops.ops.service.privacy_request.request_runner_service.upload") + def test_email_complete_send_access_no_email_identity( + self, + upload_mock, + mailgun_send, + postgres_integration_db, + postgres_example_test_dataset_config, + cache, + db, + generate_auth_header, + policy, + read_connection_config, + privacy_request_complete_email_notification_enabled, + run_privacy_request_task, + ): + upload_mock.return_value = "http://www.data-download-url" + data = { + "requested_at": "2021-08-30T16:09:37.359Z", + "policy_key": policy.key, + "identity": {"phone": "1231231233"}, + } + + pr = get_privacy_request_results( + db, + policy, + run_privacy_request_task, + data, + ) + db.refresh(pr) + assert pr.status == PrivacyRequestStatus.error + pr.delete(db=db) + + assert mailgun_send.called is False