diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b1b741a5..87cfc3881 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,13 +16,14 @@ The types of changes are: * `Fixed` for any bug fixes. * `Security` in case of vulnerabilities. - ## [Unreleased](https://github.com/ethyca/fidesops/compare/1.7.2...main) ### Changed + * Refactor privacy center to be more modular [#1363](https://github.com/ethyca/fidesops/pull/1363) ### Fixed + * Distinguish whether webhook has been visited and no fields were found, versus never visited [#1339](https://github.com/ethyca/fidesops/pull/1339) * Fix Redis Cache Early Expiration in Tests [#1358](https://github.com/ethyca/fidesops/pull/1358) * Limit values for the offset pagination strategy are now cast to integers before use [#1364](https://github.com/ethyca/fidesops/pull/1364) @@ -35,6 +36,7 @@ The types of changes are: * Allow querying the non-default schema with the Postgres Connector [#1375](https://github.com/ethyca/fidesops/pull/1375) * Frontend - ability for users to manually enter PII to an IN PROGRESS subject request [#1016](https://github.com/ethyca/fidesops/pull/1377) * Enable retries on saas connectors for failures at the http request level [#1376](https://github.com/ethyca/fidesops/pull/1376) +* Add consent request api [#1387](https://github.com/ethyca/fidesops/pull/1387) ### Removed @@ -131,6 +133,7 @@ The types of changes are: ## [1.7.1](https://github.com/ethyca/fidesops/compare/1.7.0...1.7.1) ### Breaking Changes + The `oauth2` strategy has been renamed to `oauth2_authorization_code` in order to make a distinction between the newly introduced `oauth2_client_credentials` strategy [#1159](https://github.com/ethyca/fidesops/pull/1159) ### Added @@ -143,7 +146,7 @@ The `oauth2` strategy has been renamed to `oauth2_authorization_code` in order t * SaaS Connector Configuration - Testing a Connection [#985](https://github.com/ethyca/fidesops/pull/1099) * Add an endpoint for verifying the user's identity before queuing the privacy request. [#1111](https://github.com/ethyca/fidesops/pull/1111) * Adds tests for email endpoints and service [#1112](https://github.com/ethyca/fidesops/pull/1112) -* Adds the ability to verify a subject's identity before processing a Privacy Request [#1115](https://github.com/ethyca/fidesops/pull/1115) +* Adds the ability to verify a subject's identity before processing a Privacy Request [#1115](https://github.com/ethyca/fidesops/pull/1115) * Add option to login as root user from config[#1116](https://github.com/ethyca/fidesops/pull/1116) * Added email templates [#1123](https://github.com/ethyca/fidesops/pull/1123) * Add Retry button back into the subject request detail view [#1128](https://github.com/ethyca/fidesops/pull/1131) @@ -158,7 +161,7 @@ The `oauth2` strategy has been renamed to `oauth2_authorization_code` in order t * Bump fideslib to fix issue where the authenticate button in the FastAPI docs did not work [#1092](https://github.com/ethyca/fidesops/pull/1092) * Escape the Redis user and password to make them URL friendly [#1104](https://github.com/ethyca/fidesops/pull/1104) * Reduced number of connections opened against app db during health checks [#1107](https://github.com/ethyca/fidesops/pull/1107) -* Fix FIDESOPS__ROOT_USER__ANALYTICS_ID generation when env var is set [#1113](https://github.com/ethyca/fidesops/pull/1113) +* Fix FIDESOPS**ROOT_USER**ANALYTICS_ID generation when env var is set [#1113](https://github.com/ethyca/fidesops/pull/1113) * Set localhost to None for non-endpoint events [#1130](https://github.com/ethyca/fidesops/pull/1130) * Fixed docs build in CI [#1138](https://github.com/ethyca/fidesops/pull/1138) * Added future annotations to privacy_request.py for backwards compatibility [#1136](https://github.com/ethyca/fidesops/pull/1136) @@ -471,7 +474,7 @@ The `oauth2` strategy has been renamed to `oauth2_authorization_code` in order t * GET routes for users [#405](https://github.com/ethyca/fidesops/pull/405) * Username based search on GET route [#444](https://github.com/ethyca/fidesops/pull/444) -* FIDESOPS__DEV_MODE for Easier SaaS Request Debugging [#363](https://github.com/ethyca/fidesops/pull/363) +* FIDESOPS\_\_DEV_MODE for Easier SaaS Request Debugging [#363](https://github.com/ethyca/fidesops/pull/363) * Track user privileges across sessions [#425](https://github.com/ethyca/fidesops/pull/425) * Add first_name and last_name fields. Also add them along with created_at to FidesUser response [#465](https://github.com/ethyca/fidesops/pull/465) * Denial reasons for DSR and user `AuditLog` [#463](https://github.com/ethyca/fidesops/pull/463) diff --git a/docs/fidesops/docs/postman/Fidesops.postman_collection.json b/docs/fidesops/docs/postman/Fidesops.postman_collection.json index 0c4487470..adb6e11c1 100644 --- a/docs/fidesops/docs/postman/Fidesops.postman_collection.json +++ b/docs/fidesops/docs/postman/Fidesops.postman_collection.json @@ -1,9 +1,8 @@ { "info": { - "_postman_id": "296aa41f-49f7-4988-bbfb-b57c480a695f", + "_postman_id": "8f48b8e3-0a39-4e5e-b505-8087064dc1af", "name": "Fidesops", - "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", - "_exporter_id": "1984786" + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" }, "item": [ { @@ -940,35 +939,6 @@ } }, "response": [] - }, - { - "name": "Restart failed node", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "url": { - "raw": "{{host}}/privacy-request/{{privacy_request_id}}/retry", - "host": [ - "{{host}}" - ], - "path": [ - "privacy-request", - "{{privacy_request_id}}", - "retry" - ] - } - }, - "response": [] } ] }, @@ -4443,6 +4413,40 @@ } ] }, + { + "name": "Restart from Failure", + "item": [ + { + "name": "Restart failed node", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "url": { + "raw": "{{host}}/privacy-request/{{privacy_request_id}}/retry", + "host": [ + "{{host}}" + ], + "path": [ + "privacy-request", + "{{privacy_request_id}}", + "retry" + ] + } + }, + "response": [] + } + ] + }, { "name": "ConnectionType", "item": [ @@ -4753,6 +4757,116 @@ "response": [] } ] + }, + { + "name": "Consent Request", + "item": [ + { + "name": "Create Verification Code for Consent Request", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"phone_number\": \"{{phone_number}}\",\n \"email\": \"{{email}}\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/consent-request", + "host": [ + "{{host}}" + ], + "path": [ + "consent-request" + ] + } + }, + "response": [] + }, + { + "name": "Verify Code and Return Current Preferences", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"code\": \"{{verification_code}}\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/consent-request/{{consent_request_id}}/verify", + "host": [ + "{{host}}" + ], + "path": [ + "consent-request", + "{{consent_request_id}}", + "verify" + ] + } + }, + "response": [] + }, + { + "name": "Verify Code and Save Preferences", + "request": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"identity\": {\n \"phone_number\": \"{{phone_number}}\",\n \"email\": \"{{email}}\"\n },\n \"consent\": [\n {\n \"data_use\": \"{{data_use}}\",\n \"data_use_description\": \"{{data_use_description}}\",\n \"opt_in\": {{opt_in}}\n }\n ],\n \"code\": \"{{verification_code}}\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/consent-request/{{consent_request_id}}/preferences", + "host": [ + "{{host}}" + ], + "path": [ + "consent-request", + "{{consent_request_id}}", + "preferences" + ] + } + }, + "response": [] + } + ], + "auth": { + "type": "noauth" + }, + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + } + ] } ], "event": [ @@ -5020,10 +5134,30 @@ "value": "manual_webhook_key", "type": "string" }, + { + "key": "consent_request_id", + "value": "", + "type": "default" + }, { "key": "timescale_key", "value": "", "type": "string" + }, + { + "key": "phone_number", + "value": "", + "type": "string" + }, + { + "key": "email", + "value": "", + "type": "string" + }, + { + "key": "verification_code", + "value": "", + "type": "string" } ] -} \ No newline at end of file +} diff --git a/src/fidesops/ops/api/v1/api.py b/src/fidesops/ops/api/v1/api.py index 2c8435930..1002767f2 100644 --- a/src/fidesops/ops/api/v1/api.py +++ b/src/fidesops/ops/api/v1/api.py @@ -2,6 +2,7 @@ config_endpoints, connection_endpoints, connection_type_endpoints, + consent_request_endpoints, dataset_endpoints, drp_endpoints, email_endpoints, @@ -25,6 +26,7 @@ api_router.include_router(config_endpoints.router) api_router.include_router(connection_type_endpoints.router) api_router.include_router(connection_endpoints.router) +api_router.include_router(consent_request_endpoints.router) api_router.include_router(dataset_endpoints.router) api_router.include_router(drp_endpoints.router) api_router.include_router(encryption_endpoints.router) diff --git a/src/fidesops/ops/api/v1/endpoints/consent_request_endpoints.py b/src/fidesops/ops/api/v1/endpoints/consent_request_endpoints.py new file mode 100644 index 000000000..0fd8461b9 --- /dev/null +++ b/src/fidesops/ops/api/v1/endpoints/consent_request_endpoints.py @@ -0,0 +1,238 @@ +from __future__ import annotations + +import logging + +from fastapi import Depends, HTTPException +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session +from starlette.status import ( + HTTP_200_OK, + HTTP_400_BAD_REQUEST, + HTTP_403_FORBIDDEN, + HTTP_404_NOT_FOUND, + HTTP_500_INTERNAL_SERVER_ERROR, +) + +from fidesops.ops.api.deps import get_db +from fidesops.ops.api.v1.urn_registry import ( + CONSENT_REQUEST, + CONSENT_REQUEST_PREFERENCES, + CONSENT_REQUEST_VERIFY, + V1_URL_PREFIX, +) +from fidesops.ops.common_exceptions import ( + EmailDispatchException, + FunctionalityNotConfigured, + IdentityVerificationException, +) +from fidesops.ops.core.config import config +from fidesops.ops.models.privacy_request import ( + Consent, + ConsentRequest, + ProvidedIdentity, + ProvidedIdentityType, +) +from fidesops.ops.schemas.privacy_request import Consent as ConsentSchema +from fidesops.ops.schemas.privacy_request import ( + ConsentPreferences, + ConsentPreferencesWithVerificationCode, + ConsentRequestResponse, + VerificationCode, +) +from fidesops.ops.schemas.redis_cache import Identity +from fidesops.ops.service._verification import send_verification_code_to_user +from fidesops.ops.util.api_router import APIRouter +from fidesops.ops.util.logger import Pii + +router = APIRouter(tags=["Consent"], prefix=V1_URL_PREFIX) + +logger = logging.getLogger(__name__) + + +@router.post( + CONSENT_REQUEST, + status_code=HTTP_200_OK, + response_model=ConsentRequestResponse, +) +def create_consent_request( + *, + db: Session = Depends(get_db), + data: Identity, +) -> ConsentRequestResponse: + """Creates a verification code for the user to verify access to manage consent preferences.""" + if not config.redis.enabled: + raise FunctionalityNotConfigured( + "Application redis cache required, but it is currently disabled! Please update your application configuration to enable integration with a redis cache." + ) + + if not config.execution.subject_identity_verification_required: + raise FunctionalityNotConfigured( + "Subject identity verification is required, but it is currently disabled! Please update your application configuration to enable subject identity verification." + ) + + if not data.email: + raise HTTPException(HTTP_400_BAD_REQUEST, detail="An email address is required") + + identity = ProvidedIdentity.filter( + db=db, + conditions=( + (ProvidedIdentity.field_name == ProvidedIdentityType.email) + & (ProvidedIdentity.hashed_value == ProvidedIdentity.hash_value(data.email)) + & (ProvidedIdentity.privacy_request_id.is_(None)) + ), + ).first() + + if not identity: + provided_identity_data = { + "privacy_request_id": None, + "field_name": "email", + "hashed_value": ProvidedIdentity.hash_value(data.email), + "encrypted_value": {"value": data.email}, + } + identity = ProvidedIdentity.create(db, data=provided_identity_data) + + consent_request_data = { + "provided_identity_id": identity.id, + } + consent_request = ConsentRequest.create(db, data=consent_request_data) + try: + send_verification_code_to_user(db, consent_request, data.email) + except EmailDispatchException as exc: + logger.error("Error sending the verification code email: %s", str(exc)) + raise HTTPException( + status_code=HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Error sending the verification code email: {str(exc)}", + ) + return ConsentRequestResponse( + identity=data, + consent_request_id=consent_request.id, + ) + + +@router.post( + CONSENT_REQUEST_VERIFY, + status_code=HTTP_200_OK, + response_model=ConsentPreferences, +) +def consent_request_verify( + *, + consent_request_id: str, + db: Session = Depends(get_db), + data: VerificationCode, +) -> ConsentPreferences: + """Verifies the verification code and returns the current consent preferences if successful.""" + provided_identity = _get_consent_request_and_provided_identity( + db=db, consent_request_id=consent_request_id, verification_code=data.code + ) + + if not provided_identity.hashed_value: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, detail="Provided identity missing email" + ) + + return _prepare_consent_preferences(db, provided_identity) + + +@router.patch( + CONSENT_REQUEST_PREFERENCES, + status_code=HTTP_200_OK, + response_model=ConsentPreferences, +) +def set_consent_preferences( + *, + consent_request_id: str, + db: Session = Depends(get_db), + data: ConsentPreferencesWithVerificationCode, +) -> ConsentPreferences: + """Verifies the verification code and saves the user's consent preferences if successful.""" + provided_identity = _get_consent_request_and_provided_identity( + db=db, + consent_request_id=consent_request_id, + verification_code=data.code, + ) + + if not provided_identity.hashed_value: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, detail="Provided identity missing email" + ) + + for preference in data.consent: + current_preference = Consent.filter( + db=db, + conditions=(Consent.provided_identity_id == provided_identity.id) + & (Consent.data_use == preference.data_use), + ).first() + + if current_preference: + current_preference.update(db, data=dict(preference)) + else: + preference_dict = dict(preference) + preference_dict["provided_identity_id"] = provided_identity.id + try: + Consent.create(db, data=preference_dict) + except IntegrityError as exc: + raise HTTPException( + status_code=HTTP_400_BAD_REQUEST, detail=Pii(str(exc)) + ) + + return _prepare_consent_preferences(db, provided_identity) + + +def _get_consent_request_and_provided_identity( + db: Session, + consent_request_id: str, + verification_code: str, +) -> ProvidedIdentity: + """Verifies the consent request and verification code, then return the ProvidedIdentity if successful.""" + consent_request = ConsentRequest.get_by_key_or_id( + db=db, data={"id": consent_request_id} + ) + + if not consent_request: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, detail="Consent request not found" + ) + + try: + consent_request.verify_identity(verification_code) + except IdentityVerificationException as exc: + raise HTTPException(status_code=HTTP_400_BAD_REQUEST, detail=exc.message) + except PermissionError as exc: + logger.info("Invalid verification code provided for %s.", consent_request.id) + raise HTTPException(status_code=HTTP_403_FORBIDDEN, detail=exc.args[0]) + + provided_identity: ProvidedIdentity | None = ProvidedIdentity.get_by_key_or_id( + db, data={"id": consent_request.provided_identity_id} + ) + + # It shouldn't be possible to hit this because the cascade delete of the identity + # data would also delete the consent_request, but including this as a safety net. + if not provided_identity: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail="No identity found for consent request id", + ) + + return provided_identity + + +def _prepare_consent_preferences( + db: Session, provided_identity: ProvidedIdentity +) -> ConsentPreferences: + consent = Consent.filter( + db=db, conditions=Consent.provided_identity_id == provided_identity.id + ).all() + + if not consent: + return ConsentPreferences(consent=None) + + return ConsentPreferences( + consent=[ + ConsentSchema( + data_use=x.data_use, + data_use_description=x.data_use_description, + opt_in=x.opt_in, + ) + for x in consent + ], + ) diff --git a/src/fidesops/ops/api/v1/endpoints/privacy_request_endpoints.py b/src/fidesops/ops/api/v1/endpoints/privacy_request_endpoints.py index 4cd891f45..ff6ceb8e3 100644 --- a/src/fidesops/ops/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fidesops/ops/api/v1/endpoints/privacy_request_endpoints.py @@ -73,7 +73,6 @@ from fidesops.ops.graph.traversal import Traversal from fidesops.ops.models.connectionconfig import ConnectionConfig from fidesops.ops.models.datasetconfig import DatasetConfig -from fidesops.ops.models.email import EmailConfig from fidesops.ops.models.manual_webhook import AccessManualWebhook from fidesops.ops.models.policy import ActionType, CurrentStep, Policy, PolicyPreWebhook from fidesops.ops.models.privacy_request import ( @@ -92,7 +91,6 @@ FidesopsEmail, RequestReceiptBodyParams, RequestReviewDenyBodyParams, - SubjectIdentityVerificationBodyParams, ) from fidesops.ops.schemas.external_https import PrivacyRequestResumeFormat from fidesops.ops.schemas.privacy_request import ( @@ -109,12 +107,9 @@ RowCountRequest, VerificationCode, ) -from fidesops.ops.service.email.email_dispatch_service import ( - dispatch_email, - dispatch_email_task, -) +from fidesops.ops.service._verification import send_verification_code_to_user +from fidesops.ops.service.email.email_dispatch_service import dispatch_email_task from fidesops.ops.service.privacy_request.request_runner_service import ( - generate_id_verification_code, queue_privacy_request, ) from fidesops.ops.service.privacy_request.request_service import ( @@ -236,7 +231,7 @@ async def create_privacy_request( ) if config.execution.subject_identity_verification_required: - _send_verification_code_to_user( + send_verification_code_to_user( db, privacy_request, privacy_request_data.identity.email ) created.append(privacy_request) @@ -287,27 +282,6 @@ async def create_privacy_request( ) -def _send_verification_code_to_user( - db: Session, privacy_request: PrivacyRequest, email: Optional[str] -) -> None: - """Generate and cache a verification code, and then email to the user""" - EmailConfig.get_configuration( - db=db - ) # Validates Fidesops is currently configured to send emails - verification_code: str = generate_id_verification_code() - privacy_request.cache_identity_verification_code(verification_code) - # synchronous call for now since failure to send verification code is fatal to request - dispatch_email( - db=db, - action_type=EmailActionType.SUBJECT_IDENTITY_VERIFICATION, - to_email=email, - email_body_params=SubjectIdentityVerificationBodyParams( - verification_code=verification_code, - verification_code_ttl_seconds=config.redis.identity_verification_code_ttl_seconds, - ), - ) - - def _send_privacy_request_receipt_email_to_user( policy: Optional[Policy], email: Optional[str] ) -> None: diff --git a/src/fidesops/ops/api/v1/urn_registry.py b/src/fidesops/ops/api/v1/urn_registry.py index 87ed9ea8e..eab71907a 100644 --- a/src/fidesops/ops/api/v1/urn_registry.py +++ b/src/fidesops/ops/api/v1/urn_registry.py @@ -5,6 +5,12 @@ # Config URLs CONFIG = "/config" +# Consent request URLs +CONSENT_REQUEST = "/consent-request" +CONSENT_REQUEST_PREFERENCES = "/consent-request/{consent_request_id}/preferences" +CONSENT_REQUEST_VERIFY = "/consent-request/{consent_request_id}/verify" + + # Oauth Client URLs TOKEN = "/oauth/token" CLIENT = "/oauth/client" diff --git a/src/fidesops/ops/migrations/versions/c4df5d585029_data_use_unique_together.py b/src/fidesops/ops/migrations/versions/c4df5d585029_data_use_unique_together.py new file mode 100644 index 000000000..174114e35 --- /dev/null +++ b/src/fidesops/ops/migrations/versions/c4df5d585029_data_use_unique_together.py @@ -0,0 +1,31 @@ +"""Data use unique together + +Revision ID: c4df5d585029 +Revises: cf88efa1ad89 +Create Date: 2022-09-26 23:12:00.816657 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "c4df5d585029" +down_revision = "cf88efa1ad89" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint("consent_data_use_key", "consent", type_="unique") + op.create_unique_constraint( + "uix_identity_data_use", "consent", ["provided_identity_id", "data_use"] + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint("uix_identity_data_use", "consent", type_="unique") + op.create_unique_constraint("consent_data_use_key", "consent", ["data_use"]) + # ### end Alembic commands ### diff --git a/src/fidesops/ops/models/privacy_request.py b/src/fidesops/ops/models/privacy_request.py index d3ffb3c0b..be5df5d7d 100644 --- a/src/fidesops/ops/models/privacy_request.py +++ b/src/fidesops/ops/models/privacy_request.py @@ -18,7 +18,7 @@ from fideslib.oauth.jwt import generate_jwe from sqlalchemy import Boolean, Column, DateTime from sqlalchemy import Enum as EnumColumn -from sqlalchemy import ForeignKey, String +from sqlalchemy import ForeignKey, String, UniqueConstraint from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.ext.mutable import MutableDict, MutableList from sqlalchemy.orm import Session, backref, relationship @@ -787,12 +787,14 @@ class Consent(Base): provided_identity_id = Column( String, ForeignKey(ProvidedIdentity.id), nullable=False ) - data_use = Column(String, nullable=False, unique=True) + data_use = Column(String, nullable=False) data_use_description = Column(String) opt_in = Column(Boolean, nullable=False) provided_identity = relationship(ProvidedIdentity, back_populates="consent") + UniqueConstraint(provided_identity_id, data_use, name="uix_identity_data_use") + class ConsentRequest(Base): """Tracks consent requests.""" @@ -806,6 +808,44 @@ class ConsentRequest(Base): back_populates="consent_request", ) + def cache_identity_verification_code(self, value: str) -> None: + """Cache the generated identity verification code for later comparison.""" + cache: FidesopsRedis = get_cache() + cache.set_with_autoexpire( + f"IDENTITY_VERIFICATION_CODE__{self.id}", + value, + config.redis.identity_verification_code_ttl_seconds, + ) + + def get_cached_identity_data(self) -> Dict[str, Any]: + """Retrieves any identity data pertaining to this request from the cache.""" + prefix = f"id-{self.id}-identity-*" + cache: FidesopsRedis = get_cache() + keys = cache.keys(prefix) + return {key.split("-")[-1]: cache.get(key) for key in keys} + + def get_cached_verification_code(self) -> Optional[str]: + """Retrieve the generated identity verification code if it exists""" + cache = get_cache() + values = cache.get_values([f"IDENTITY_VERIFICATION_CODE__{self.id}"]) or {} + if not values: + return None + + return values.get(f"IDENTITY_VERIFICATION_CODE__{self.id}", None) + + def verify_identity(self, provided_code: str) -> ConsentRequest: + """Verify the identification code supplied by the user.""" + code: Optional[str] = self.get_cached_verification_code() + if not code: + raise IdentityVerificationException( + f"Identification code expired for {self.id}." + ) + + if code != provided_code: + raise PermissionError(f"Incorrect identification code for '{self.id}'") + + return self + # Unique text to separate a step from a collection address, so we can store two values in one. PAUSED_SEPARATOR = "__fidesops_paused_sep__" diff --git a/src/fidesops/ops/schemas/privacy_request.py b/src/fidesops/ops/schemas/privacy_request.py index 479cc5039..b112bb1cd 100644 --- a/src/fidesops/ops/schemas/privacy_request.py +++ b/src/fidesops/ops/schemas/privacy_request.py @@ -214,3 +214,39 @@ class BulkPostPrivacyRequests(BulkResponse): class BulkReviewResponse(BulkPostPrivacyRequests): """Schema with mixed success/failure responses for Bulk Approve/Deny of PrivacyRequest responses.""" + + +class Consent(BaseSchema): + """Schema for consent.""" + + data_use: str + data_use_description: Optional[str] = None + opt_in: bool + + +class ConsentPreferences(BaseSchema): + """Schema for consent prefernces.""" + + consent: Optional[List[Consent]] = None + + +class ConsentPreferencesWithVerificationCode(BaseSchema): + """scheam for consent preferences including the verification code.""" + + code: str + consent: List[Consent] + + +class ConsentRequestResponse(BaseSchema): + """Schema for consent request response.""" + + consent_request_id: str + + +class ConsentRequestVerification(BaseSchema): + """Schema for consent requests.""" + + identity: Identity + data_use: str + data_use_description: Optional[str] = None + opt_in: bool diff --git a/src/fidesops/ops/service/_verification.py b/src/fidesops/ops/service/_verification.py new file mode 100644 index 000000000..b0b067ba9 --- /dev/null +++ b/src/fidesops/ops/service/_verification.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from sqlalchemy.orm import Session + +from fidesops.ops.core.config import config +from fidesops.ops.models.email import EmailConfig +from fidesops.ops.models.privacy_request import ConsentRequest, PrivacyRequest +from fidesops.ops.schemas.email.email import ( + EmailActionType, + SubjectIdentityVerificationBodyParams, +) +from fidesops.ops.service.email.email_dispatch_service import dispatch_email +from fidesops.ops.service.privacy_request.request_runner_service import ( + generate_id_verification_code, +) + + +def send_verification_code_to_user( + db: Session, request: ConsentRequest | PrivacyRequest, email: str | None +) -> str: + """Generate and cache a verification code, and then email to the user""" + EmailConfig.get_configuration( + db=db + ) # Validates Fidesops is currently configured to send emails + verification_code = generate_id_verification_code() + request.cache_identity_verification_code(verification_code) + dispatch_email( + db, + action_type=EmailActionType.SUBJECT_IDENTITY_VERIFICATION, + to_email=email, + email_body_params=SubjectIdentityVerificationBodyParams( + verification_code=verification_code, + verification_code_ttl_seconds=config.redis.identity_verification_code_ttl_seconds, + ), + ) + + return verification_code diff --git a/src/fidesops/ops/service/email/email_dispatch_service.py b/src/fidesops/ops/service/email/email_dispatch_service.py index 908a1e224..7f81940d9 100644 --- a/src/fidesops/ops/service/email/email_dispatch_service.py +++ b/src/fidesops/ops/service/email/email_dispatch_service.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import logging from typing import Any, Dict, List, Optional, Union diff --git a/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py b/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py new file mode 100644 index 000000000..5ca3f9605 --- /dev/null +++ b/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py @@ -0,0 +1,360 @@ +from __future__ import annotations + +from copy import deepcopy +from typing import Any +from unittest.mock import patch + +import pytest + +from fidesops.ops.api.v1.urn_registry import ( + CONSENT_REQUEST, + CONSENT_REQUEST_PREFERENCES, + CONSENT_REQUEST_VERIFY, + V1_URL_PREFIX, +) +from fidesops.ops.core.config import config +from fidesops.ops.models.privacy_request import ( + Consent, + ConsentRequest, + ProvidedIdentity, +) + + +@pytest.fixture +def provided_identity_and_consent_request(db): + provided_identity_data = { + "privacy_request_id": None, + "field_name": "email", + "hashed_value": ProvidedIdentity.hash_value("test@email.com"), + "encrypted_value": {"value": "test@email.com"}, + } + provided_identity = ProvidedIdentity.create(db, data=provided_identity_data) + + consent_request_data = { + "provided_identity_id": provided_identity.id, + } + consent_request = ConsentRequest.create(db, data=consent_request_data) + + yield provided_identity, consent_request + + +@pytest.fixture +def disable_redis(): + current = config.redis.enabled + config.redis.enabled = False + yield + config.redis.enabled = current + + +class TestConsentRequest: + @pytest.mark.usefixtures( + "email_config", + "email_connection_config", + "email_dataset_config", + "subject_identity_verification_required", + ) + @patch("fidesops.ops.service._verification.dispatch_email") + def test_consent_request(self, mock_dispatch_email, api_client): + data = {"email": "test@example.com"} + response = api_client.post(f"{V1_URL_PREFIX}{CONSENT_REQUEST}", json=data) + assert response.status_code == 200 + assert mock_dispatch_email.called + + @pytest.mark.usefixtures( + "email_config", + "email_connection_config", + "email_dataset_config", + "subject_identity_verification_required", + ) + @patch("fidesops.ops.service._verification.dispatch_email") + def test_consent_request_identity_present( + self, mock_dispatch_email, provided_identity_and_consent_request, api_client + ): + provided_identity, _ = provided_identity_and_consent_request + data = {"email": provided_identity.encrypted_value["value"]} + response = api_client.post(f"{V1_URL_PREFIX}{CONSENT_REQUEST}", json=data) + assert response.status_code == 200 + assert mock_dispatch_email.called + + @pytest.mark.usefixtures( + "email_config", + "email_connection_config", + "email_dataset_config", + "subject_identity_verification_required", + "disable_redis", + ) + def test_consent_request_redis_disabled(self, api_client): + data = {"email": "test@example.com"} + response = api_client.post(f"{V1_URL_PREFIX}{CONSENT_REQUEST}", json=data) + assert response.status_code == 500 + assert "redis cache required" in response.json()["message"] + + @pytest.mark.usefixtures( + "email_config", + "email_connection_config", + "email_dataset_config", + ) + def test_consent_request_subject_verification_disabled(self, api_client): + data = {"email": "test@example.com"} + response = api_client.post(f"{V1_URL_PREFIX}{CONSENT_REQUEST}", json=data) + assert response.status_code == 500 + assert "identity verification" in response.json()["message"] + + @pytest.mark.usefixtures( + "email_config", + "email_connection_config", + "email_dataset_config", + "subject_identity_verification_required", + ) + def test_consent_request_no_email(self, api_client): + data = {"phone_number": "336-867-5309"} + response = api_client.post(f"{V1_URL_PREFIX}{CONSENT_REQUEST}", json=data) + assert response.status_code == 400 + assert "email address is required" in response.json()["detail"] + + +class TestConsentVerify: + def test_consent_verify_no_consent_request_id( + self, + api_client, + ): + data = {"code": "12345"} + + response = api_client.post( + f"{V1_URL_PREFIX}{CONSENT_REQUEST_VERIFY.format(consent_request_id='abcd')}", + json=data, + ) + assert response.status_code == 404 + assert "not found" in response.json()["detail"] + + def test_consent_verify_no_consent_code( + self, provided_identity_and_consent_request, api_client + ): + data = {"code": "12345"} + + _, consent_request = provided_identity_and_consent_request + response = api_client.post( + f"{V1_URL_PREFIX}{CONSENT_REQUEST_VERIFY.format(consent_request_id=consent_request.id)}", + json=data, + ) + assert response.status_code == 400 + assert "code expired" in response.json()["detail"] + + def test_consent_verify_invalid_code( + self, provided_identity_and_consent_request, api_client + ): + _, consent_request = provided_identity_and_consent_request + consent_request.cache_identity_verification_code("abcd") + + response = api_client.post( + f"{V1_URL_PREFIX}{CONSENT_REQUEST_VERIFY.format(consent_request_id=consent_request.id)}", + json={"code": "1234"}, + ) + assert response.status_code == 403 + assert "Incorrect identification" in response.json()["detail"] + + def test_consent_verify_no_email_provided(self, db, api_client): + provided_identity_data = { + "privacy_request_id": None, + "field_name": "email", + "hashed_value": None, + "encrypted_value": None, + } + provided_identity = ProvidedIdentity.create(db, data=provided_identity_data) + + consent_request_data = { + "provided_identity_id": provided_identity.id, + } + consent_request = ConsentRequest.create(db, data=consent_request_data) + verification_code = "abcd" + consent_request.cache_identity_verification_code(verification_code) + + response = api_client.post( + f"{V1_URL_PREFIX}{CONSENT_REQUEST_VERIFY.format(consent_request_id=consent_request.id)}", + json={"code": verification_code}, + ) + + assert response.status_code == 404 + assert "missing email" in response.json()["detail"] + + def test_consent_verify_no_consent_present( + self, provided_identity_and_consent_request, api_client + ): + _, consent_request = provided_identity_and_consent_request + verification_code = "abcd" + consent_request.cache_identity_verification_code(verification_code) + + response = api_client.post( + f"{V1_URL_PREFIX}{CONSENT_REQUEST_VERIFY.format(consent_request_id=consent_request.id)}", + json={"code": verification_code}, + ) + assert response.status_code == 200 + assert response.json()["consent"] is None + + def test_consent_verify_consent_preferences( + self, provided_identity_and_consent_request, db, api_client + ): + verification_code = "abcd" + provided_identity, consent_request = provided_identity_and_consent_request + consent_request.cache_identity_verification_code(verification_code) + + consent_data: list[dict[str, Any]] = [ + { + "data_use": "email", + "data_use_description": None, + "opt_in": True, + }, + { + "data_use": "location", + "data_use_description": "Location data", + "opt_in": False, + }, + ] + + for data in deepcopy(consent_data): + data["provided_identity_id"] = provided_identity.id + Consent.create(db, data=data) + + response = api_client.post( + f"{V1_URL_PREFIX}{CONSENT_REQUEST_VERIFY.format(consent_request_id=consent_request.id)}", + json={"code": verification_code}, + ) + assert response.status_code == 200 + assert response.json()["consent"] == consent_data + + +class TestSaveConsent: + def test_set_consent_preferences_no_consent_request_id(self, api_client): + data = { + "code": "12345", + "identity": {"email": "test@email.com"}, + "consent": [{"data_use": "email", "opt_in": True}], + } + + response = api_client.patch( + f"{V1_URL_PREFIX}{CONSENT_REQUEST_PREFERENCES.format(consent_request_id='abcd')}", + json=data, + ) + assert response.status_code == 404 + assert "not found" in response.json()["detail"] + + def test_set_consent_preferences_no_consent_code( + self, provided_identity_and_consent_request, api_client + ): + _, consent_request = provided_identity_and_consent_request + + data = { + "code": "12345", + "identity": {"email": "test@email.com"}, + "consent": [{"data_use": "email", "opt_in": True}], + } + + response = api_client.patch( + f"{V1_URL_PREFIX}{CONSENT_REQUEST_PREFERENCES.format(consent_request_id=consent_request.id)}", + json=data, + ) + assert response.status_code == 400 + assert "code expired" in response.json()["detail"] + + def test_set_consent_preferences_invalid_code( + self, provided_identity_and_consent_request, api_client + ): + _, consent_request = provided_identity_and_consent_request + consent_request.cache_identity_verification_code("abcd") + + data = { + "code": "12345", + "identity": {"email": "test@email.com"}, + "consent": [{"data_use": "email", "opt_in": True}], + } + response = api_client.patch( + f"{V1_URL_PREFIX}{CONSENT_REQUEST_PREFERENCES.format(consent_request_id=consent_request.id)}", + json=data, + ) + assert response.status_code == 403 + assert "Incorrect identification" in response.json()["detail"] + + def test_set_consent_preferences_no_email_provided(self, db, api_client): + provided_identity_data = { + "privacy_request_id": None, + "field_name": "email", + "hashed_value": None, + "encrypted_value": None, + } + provided_identity = ProvidedIdentity.create(db, data=provided_identity_data) + + consent_request_data = { + "provided_identity_id": provided_identity.id, + } + consent_request = ConsentRequest.create(db, data=consent_request_data) + verification_code = "abcd" + consent_request.cache_identity_verification_code(verification_code) + + data = { + "code": verification_code, + "identity": {"email": "test@email.com"}, + "consent": [{"data_use": "email", "opt_in": True}], + } + response = api_client.patch( + f"{V1_URL_PREFIX}{CONSENT_REQUEST_PREFERENCES.format(consent_request_id=consent_request.id)}", + json=data, + ) + + assert response.status_code == 404 + assert "missing email" in response.json()["detail"] + + def test_set_consent_preferences_no_consent_present( + self, provided_identity_and_consent_request, api_client + ): + _, consent_request = provided_identity_and_consent_request + verification_code = "abcd" + consent_request.cache_identity_verification_code(verification_code) + + data = { + "code": verification_code, + "identity": {"email": "test@email.com"}, + "consent": None, + } + response = api_client.patch( + f"{V1_URL_PREFIX}{CONSENT_REQUEST_PREFERENCES.format(consent_request_id=consent_request.id)}", + json=data, + ) + assert response.status_code == 422 + + def test_set_consent_consent_preferences( + self, provided_identity_and_consent_request, db, api_client + ): + provided_identity, consent_request = provided_identity_and_consent_request + verification_code = "abcd" + consent_request.cache_identity_verification_code(verification_code) + + consent_data: list[dict[str, Any]] = [ + { + "data_use": "email", + "data_use_description": None, + "opt_in": True, + }, + { + "data_use": "location", + "data_use_description": "Location data", + "opt_in": False, + }, + ] + + for data in deepcopy(consent_data): + data["provided_identity_id"] = provided_identity.id + Consent.create(db, data=data) + + consent_data[1]["opt_in"] = False + + data = { + "code": verification_code, + "identity": {"email": "test@email.com"}, + "consent": consent_data, + } + response = api_client.patch( + f"{V1_URL_PREFIX}{CONSENT_REQUEST_PREFERENCES.format(consent_request_id=consent_request.id)}", + json=data, + ) + assert response.status_code == 200 + assert response.json()["consent"] == consent_data diff --git a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py index cebb53ce5..50a1484b9 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py @@ -2957,9 +2957,7 @@ def test_create_privacy_request_no_email_config( @mock.patch( "fidesops.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) - @mock.patch( - "fidesops.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_email" - ) + @mock.patch("fidesops.ops.service._verification.dispatch_email") def test_create_privacy_request_with_email_config( self, mock_dispatch_email, diff --git a/tests/ops/conftest.py b/tests/ops/conftest.py index b8645b156..6436c4d39 100644 --- a/tests/ops/conftest.py +++ b/tests/ops/conftest.py @@ -248,6 +248,15 @@ def require_manual_request_approval(): config.execution.require_manual_request_approval = original_value +@pytest.fixture(scope="function") +def subject_identity_verification_required(): + """Enable identity verification.""" + original_value = config.execution.subject_identity_verification_required + config.execution.subject_identity_verification_required = True + yield + config.execution.subject_identity_verification_required = original_value + + @pytest.fixture(autouse=True, scope="function") def subject_identity_verification_not_required(): """Disable identity verification for most tests unless overridden"""