diff --git a/CHANGELOG.md b/CHANGELOG.md index 112167324..40caaf0c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,9 @@ The types of changes are: ## [Unreleased](https://github.com/ethyca/fidesops/compare/1.7.1...main) +### Added +* Adds users and owners configuration for Hubspot connector [#1091](https://github.com/ethyca/fidesops/pull/1091) + ## [1.7.1](https://github.com/ethyca/fidesops/compare/1.7.0...1.7.1) ### Added @@ -50,7 +53,7 @@ The types of changes are: * Fix FIDESOPS__ROOT_USER__ANALYTICS_ID generation when env var is set [#1113](https://github.com/ethyca/fidesops/pull/1113) * Set localhost to None for non-endpoint events [#1130](https://github.com/ethyca/fidesops/pull/1130) * Fixed docs build in CI [#1138](https://github.com/ethyca/fidesops/pull/1138) -* Added future annotatioins to privacy_request.py for backwards compatibility [#1136](https://github.com/ethyca/fidesops/pull/1136) +* Added future annotations to privacy_request.py for backwards compatibility [#1136](https://github.com/ethyca/fidesops/pull/1136) ### Changed diff --git a/data/saas/config/hubspot_config.yml b/data/saas/config/hubspot_config.yml index 6fab80034..5f18b0041 100644 --- a/data/saas/config/hubspot_config.yml +++ b/data/saas/config/hubspot_config.yml @@ -8,20 +8,19 @@ saas_config: connector_params: - name: domain default_value: api.hubapi.com - - name: hapikey + - name: private_app_token client_config: protocol: https host: authentication: - strategy: query_param + strategy: bearer configuration: - name: hapikey - value: + token: test_request: method: GET - path: /companies/v2/companies/paged + path: /settings/v3/users endpoints: - name: contacts @@ -45,10 +44,7 @@ saas_config: param_values: - name: email identity: email - postprocessors: - - strategy: unwrap - configuration: - data_path: results + data_path: results pagination: strategy: link configuration: @@ -73,17 +69,18 @@ saas_config: path: /crm/v3/owners method: GET query_params: - - name: email - value: - name: limit value: 100 param_values: - - name: email + - name: placeholder identity: email + data_path: results postprocessors: - - strategy: unwrap + - strategy: filter configuration: - data_path: results + field: email + value: + identity: email pagination: strategy: link configuration: @@ -121,3 +118,35 @@ saas_config: configuration: field: status value: SUBSCRIBED + - name: users + requests: + read: + path: /settings/v3/users/ + method: GET + query_params: + - name: limit + value: 100 + param_values: + - name: placeholder + identity: email + data_path: results + postprocessors: + - strategy: filter + configuration: + field: email + value: + identity: email + pagination: + strategy: link + configuration: + source: body + path: paging.next.link + delete: + path: /settings/v3/users/ + method: DELETE + param_values: + - name: userId + references: + - dataset: + field: users.id + direction: from diff --git a/data/saas/dataset/hubspot_dataset.yml b/data/saas/dataset/hubspot_dataset.yml index 4d8bc9aa1..22e788ae6 100644 --- a/data/saas/dataset/hubspot_dataset.yml +++ b/data/saas/dataset/hubspot_dataset.yml @@ -104,363 +104,6 @@ dataset: data_categories: [user.sensor] fidesops_meta: data_type: string - # - name: marketing_emails - # fields: - # - name: ab - # data_categories: [system.operations] - # fidesops_meta: - # data_type: boolean - # - name: abHoursToWait - # data_categories: [system.operations] - # fidesops_meta: - # data_type: integer - # - name: abSampleSizeDefault - # data_categories: [system.operations] - # fidesops_meta: - # data_type: string - # - name: abSamplingDefault - # data_categories: [system.operations] - # fidesops_meta: - # data_type: string - # - name: abSuccessMetric - # data_categories: [system.operations] - # fidesops_meta: - # data_type: string - # - name: abTestPercentage - # data_categories: [system.operations] - # fidesops_meta: - # data_type: integer - # - name: abVariation - # data_categories: [system.operations] - # fidesops_meta: - # data_type: boolean - # - name: absoluteUrl - # data_categories: [user] - # fidesops_meta: - # data_type: string - # - name: allEmailCampaignIds - # data_categories: [system.operations] - # fidesops_meta: - # data_type: integer[] - # - name: analyticsPageId - # data_categories: [system.operations] - # fidesops_meta: - # data_type: string - # - name: analyticsPageType - # data_categories: [system.operations] - # fidesops_meta: - # data_type: string - # - name: archived - # data_categories: [system.operations] - # fidesops_meta: - # data_type: boolean - # - name: author - # data_categories: [user.contact.email] - # fidesops_meta: - # data_type: string - # - name: authorAt - # data_categories: [system.operations] - # fidesops_meta: - # data_type: integer - # - name: authorEmail - # data_categories: [user.contact.email] - # fidesops_meta: - # data_type: string - # - name: authorName - # data_categories: [user.name] - # fidesops_meta: - # data_type: string - # - name: authorUserId - # data_categories: [user.unique_id] - # fidesops_meta: - # data_type: integer - # - name: blogRssSettings - # data_categories: [system.operations] - # - name: campaign - # data_categories: [system.operations] - # fidesops_meta: - # data_type: string - # - name: campaignName - # data_categories: [user] - # fidesops_meta: - # data_type: string - # - name: campaignUtm - # data_categories: [user] - # fidesops_meta: - # data_type: string - # - name: canSpamSettingsId - # data_categories: [system.operations] - # fidesops_meta: - # data_type: integer - # - name: categoryId - # data_categories: [system.operations] - # fidesops_meta: - # data_type: integer - # - name: contentTypeCategory - # data_categories: [system.operations] - # fidesops_meta: - # data_type: integer - # - name: createPage - # data_categories: [system.operations] - # fidesops_meta: - # data_type: boolean - # - name: created - # data_categories: [user.sensor] - # fidesops_meta: - # data_type: integer - # - name: createdById - # data_categories: [system.operations] - # fidesops_meta: - # data_type: integer - # - name: currentState - # data_categories: [system.operations] - # fidesops_meta: - # data_type: string - # - name: currentlyPublished - # data_categories: [system.operations] - # fidesops_meta: - # data_type: boolean - # - name: domain - # data_categories: [user] - # fidesops_meta: - # data_type: string - # - name: emailBody - # data_categories: [user] - # fidesops_meta: - # data_type: string - # - name: emailNote - # data_categories: [user] - # fidesops_meta: - # data_type: string - # - name: emailTemplateMode - # data_categories: [system.operations] - # fidesops_meta: - # data_type: string - # - name: emailType - # data_categories: [system.operations] - # fidesops_meta: - # data_type: string - # - name: emailbodyPlaintext - # data_categories: [user] - # fidesops_meta: - # data_type: string - # - name: feedbackEmailCategory - # data_categories: [system.operations] - # fidesops_meta: - # data_type: string - # - name: feedbackSurveyId - # data_categories: [system.operations] - # fidesops_meta: - # data_type: integer - # - name: flexAreas - # data_categories: [system.operations] - # - name: freezeDate - # data_categories: [system.operations] - # fidesops_meta: - # data_type: integer - # - name: fromName - # data_categories: [user.name] - # fidesops_meta: - # data_type: string - # - name: htmlTitle - # data_categories: [user] - # fidesops_meta: - # data_type: string - # - name: id - # data_categories: [system.operations] - # fidesops_meta: - # data_type: integer - # primary_key: True - # - name: isGraymailSuppressionEnabled - # data_categories: [system.operations] - # fidesops_meta: - # data_type: boolean - # - name: isLocalTimezoneSend - # data_categories: [system.operations] - # fidesops_meta: - # data_type: boolean - # - name: isPublished - # data_categories: [system.operations] - # fidesops_meta: - # data_type: boolean - # - name: isRecipientFatigueSuppressionEnabled - # data_categories: [system.operations] - # fidesops_meta: - # data_type: boolean - # - name: lastEditSessionId - # data_categories: [system.operations] - # fidesops_meta: - # data_type: integer - # - name: lastEditUpdateId - # data_categories: [system.operations] - # fidesops_meta: - # data_type: integer - # - name: layoutSections - # data_categories: [system.operations] - # - name: leadFlowId - # data_categories: [system.operations] - # fidesops_meta: - # data_type: boolean - # - name: liveDomain - # data_categories: [user] - # fidesops_meta: - # data_type: string - # - name: mailingListsExcluded, - # data_categories: [system.operations] - # fidesops_meta: - # data_type: integer[] - # - name: mailingListsIncluded - # data_categories: [system.operations] - # fidesops_meta: - # data_type: integer[] - # - name: maxRssEntries - # data_categories: [system.operations] - # fidesops_meta: - # data_type: integer - # - name: metaDescription - # data_categories: [system.operations] - # fidesops_meta: - # data_type: string - # - name: name - # data_categories: [system.operations] - # fidesops_meta: - # data_type: string - # - name: pageExpiryEnabled - # data_categories: [system.operations] - # fidesops_meta: - # data_type: boolean - # - name: pageRedirected - # data_categories: [system.operations] - # fidesops_meta: - # data_type: boolean - # - name: portalId - # data_categories: [system.operations] - # fidesops_meta: - # data_type: integer - # - name: previewKey - # data_categories: [system.operations] - # fidesops_meta: - # data_type: string - # - name: processingStatus - # data_categories: [system.operations] - # fidesops_meta: - # data_type: string - # - name: publishDate - # data_categories: [user.sensor] - # fidesops_meta: - # data_type: integer - # - name: publishImmediately - # data_categories: [system.operations] - # fidesops_meta: - # data_type: boolean - # - name: publishedUrl - # data_categories: [system.operations] - # fidesops_meta: - # data_type: string - # - name: replyTo - # data_categories: [user.contact.email] - # fidesops_meta: - # data_type: string - # - name: resolvedDomain - # data_categories: [user] - # fidesops_meta: - # data_type: string - # - name: rssEmailByText - # data_categories: [user] - # fidesops_meta: - # data_type: string - # - name: rssEmailClickThroughText - # data_categories: [user] - # fidesops_meta: - # data_type: string - # - name: rssEmailCommentText - # data_categories: [user] - # fidesops_meta: - # data_type: string - # - name: rssEmailEntryTemplateEnabled - # data_categories: [system.operations] - # fidesops_meta: - # data_type: boolean - # - name: rssEmailImageMaxWidth - # data_categories: [system.operations] - # fidesops_meta: - # data_type: integer - # - name: rssEmailUrl - # data_categories: [user] - # fidesops_meta: - # data_type: string - # - name: scrubsSubscriptionLinks - # data_categories: [system.operations] - # fidesops_meta: - # data_type: boolean - # - name: slug - # data_categories: [user] - # fidesops_meta: - # data_type: string - # - name: smartEmailFields - # data_categories: [system.operations] - # - name: state - # data_categories: [system.operations] - # fidesops_meta: - # data_type: string - # - name: styleSettings - # data_categories: [system.operations] - # fidesops_meta: - # data_type: string - # - name: subcategory - # data_categories: [system.operations] - # fidesops_meta: - # data_type: string - # - name: subject - # data_categories: [user] - # fidesops_meta: - # data_type: string - # - name: subscription - # data_categories: [system.operations] - # fidesops_meta: - # data_type: integer - # - name: subscriptionName - # data_categories: [user] - # fidesops_meta: - # data_type: string - # - name: teamPerms - # data_categories: [system.operations] - # - name: templatePath - # data_categories: [system.operations] - # fidesops_meta: - # data_type: string - # - name: transactional - # data_categories: [system.operations] - # fidesops_meta: - # data_type: boolean - # - name: unpublishedAt - # data_categories: [user.sensor] - # fidesops_meta: - # data_type: integer - # - name: updated - # data_categories: [user.sensor] - # fidesops_meta: - # data_type: integer - # - name: updatedById - # data_categories: [system.operations] - # fidesops_meta: - # data_type: integer - # - name: url - # data_categories: [user] - # fidesops_meta: - # data_type: string - # - name: useRssHeadlineAsSubject - # data_categories: [system.operations] - # fidesops_meta: - # data_type: boolean - # - name: userPerms - # data_categories: [system.operations] - # - name: vidsExcluded - # data_categories: [system.operations] - # - name: vidsIncluded - # data_categories: [system.operations] - # - name: widgets - # data_categories: [system.operations] - name: subscription_preferences fields: - name: recipient @@ -504,49 +147,26 @@ dataset: data_categories: [system.operations] fidesops_meta: data_type: string -# - name: users -# fields: -# - name: id -# data_categories: [user.unique_id] -# fidesops_meta: -# primary_key: True -# data_type: string -# - name: email -# data_categories: [user.contact.email] -# fidesops_meta: -# data_type: string -# - name: roleId -# data_categories: [system.operations] -# fidesops_meta: -# data_type: string -# - name: primaryTeamId -# data_categories: [system.operations] -# fidesops_meta: -# data_type: string -# - name: secondaryTeamIds -# data_categories: [system.operations] -# fidesops_meta: -# data_type: string[] -# - name: user_provisioning -# fields: -# - name: id -# data_categories: [user.unique_id] -# fidesops_meta: -# primary_key: True -# data_type: string -# - name: email -# data_categories: [user.contact.email] -# fidesops_meta: -# data_type: string -# - name: roleId -# data_categories: [system.operations] -# fidesops_meta: -# data_type: string -# - name: primaryTeamId -# data_categories: [system.operations] -# fidesops_meta: -# data_type: string -# - name: secondaryTeamIds -# data_categories: [system.operations] -# fidesops_meta: -# data_type: string[] + - name: users + fields: + - name: id + data_categories: [user.unique_id] + fidesops_meta: + primary_key: True + data_type: string + - name: email + data_categories: [user.contact.email] + fidesops_meta: + data_type: string + - name: roleId + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: primaryTeamId + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: secondaryTeamIds + data_categories: [system.operations] + fidesops_meta: + data_type: string[] diff --git a/docs/fidesops/docs/saas_connectors/example_configs/hubspot.md b/docs/fidesops/docs/saas_connectors/example_configs/hubspot.md index b416e7ce0..629172b95 100644 --- a/docs/fidesops/docs/saas_connectors/example_configs/hubspot.md +++ b/docs/fidesops/docs/saas_connectors/example_configs/hubspot.md @@ -9,9 +9,8 @@ Fidesops uses the following Hubspot endpoints to retrieve and delete Personally |[Search](https://developers.hubspot.com/docs/api/crm/search) | Yes | No | |[Contacts](https://developers.hubspot.com/docs/api/crm/contacts) | Yes | Yes | |[Owners](https://developers.hubspot.com/docs/api/crm/owners) | Yes | No | -|[Marketing Emails](https://developers.hubspot.com/docs/api/marketing/marketing-emails) | Yes | No | |[Communication Preferences](https://developers.hubspot.com/docs/api/marketing-api/subscriptions-preferences#endpoint?spec=POST-/communication-preferences/v3/unsubscribe) | Yes | Yes | -|[Users](https://developers.hubspot.com/docs/api/settings/user-provisioning) | Yes | No | +|[Users](https://developers.hubspot.com/docs/api/settings/user-provisioning) | Yes | Yes | @@ -25,22 +24,22 @@ Fidesops provides a [Postman collection](../../postman/using_postman.md) for eas saas_config: fides_key: hubspot_connector_example name: Hubspot SaaS Config - description: A sample schema representing the Hubspot connector for fidesops + type: hubspot + description: A sample schema representing the Hubspot connector for Fidesops version: 0.0.1 connector_params: - name: domain - - name: hapikey + default_value: api.hubapi.com + - name: private_app_token client_config: protocol: https - host: - connector_param: domain + host: authentication: - strategy: query_param + strategy: bearer configuration: - token: - connector_param: hapikey + token: test_request: method: GET @@ -52,25 +51,23 @@ saas_config: read: path: /crm/v3/objects/contacts/search method: POST - body: '{ - "filterGroups": [{ - "filters": [{ - "value": "", - "propertyName": "email", - "operator": "EQ" + body: | + { + "filterGroups": [{ + "filters": [{ + "value": "", + "propertyName": "email", + "operator": "EQ" + }] }] - }] - }' + } query_params: - name: limit value: 100 param_values: - name: email identity: email - postprocessors: - - strategy: unwrap - configuration: - data_path: results + data_path: results pagination: strategy: link configuration: @@ -79,9 +76,10 @@ saas_config: update: path: /crm/v3/objects/contacts/ method: PATCH - body: '{ + body: | + { - }' + } param_values: - name: contactId references: @@ -94,57 +92,25 @@ saas_config: path: /crm/v3/owners method: GET query_params: - - name: email - value: - name: limit value: 100 param_values: - - name: email + - name: placeholder identity: email postprocessors: - strategy: unwrap configuration: data_path: results + - strategy: filter + configuration: + field: email + value: + identity: email pagination: strategy: link configuration: source: body path: paging.next.link -# - name: marketing_emails -# requests: -# read: -# path: /marketing-emails/v1/emails -# method: GET -# query_params: -# - name: limit -# value: 100 -# - name: offset -# value: 0 -# param_values: -# - name: placeholder -# identity: email -# data_path: objects -# postprocessors: -# - strategy: filter -# configuration: -# field: authorEmail # or email? -# value: -# identity: email -# pagination: -# strategy: offset -# configuration: -# incremental_param: offset -# increment_by: 100 -# limit: 10000 -# update: -# path: marketing-emails/v1/emails/ -# method: PUT -# param_values: -# - name: emailId -# references: -# - dataset: hubspot_connector_example -# field: marketing_emails.id -# direction: from - name: subscription_preferences requests: read: @@ -156,12 +122,13 @@ saas_config: update: path: /communication-preferences/v3/unsubscribe method: POST - body: '{ - "emailAddress": "", - "subscriptionId": "", - "legalBasis": "LEGITIMATE_INTEREST_CLIENT", - "legalBasisExplanation": "At users request, we opted them out" - }' + body: | + { + "emailAddress": "", + "subscriptionId": "", + "legalBasis": "LEGITIMATE_INTEREST_CLIENT", + "legalBasisExplanation": "At users request, we opted them out" + } data_path: subscriptionStatuses param_values: - name: email @@ -176,42 +143,39 @@ saas_config: configuration: field: status value: SUBSCRIBED -# - name: users -# requests: -# read: -# path: /settings/v3/users/ -# method: GET -# query_params: -# - name: limit -# value: 100 -# param_values: -# - name: placeholder -# identity: email -# pagination: -# strategy: link -# configuration: -# source: body -# path: paging.next.link -# postprocessors: -# - strategy: unwrap -# configuration: -# data_path: results -# - strategy: filter -# configuration: -# field: email -# value: -# identity: email -# - name: user_provisioning -# requests: -# read: -# path: /settings/v3/users/ -# method: GET -# param_values: -# - name: userId -# references: -# - dataset: hubspot_connector_example -# field: users.id -# direction: from -# - name: placeholder -# identity: email -``` \ No newline at end of file + - name: users + requests: + read: + path: /settings/v3/users/ + method: GET + query_params: + - name: limit + value: 100 + param_values: + - name: placeholder + identity: email + postprocessors: + - strategy: unwrap + configuration: + data_path: results + - strategy: filter + configuration: + field: email + value: + identity: email + pagination: + strategy: link + configuration: + source: body + path: paging.next.link + delete: + path: /settings/v3/users/ + method: DELETE + param_values: + - name: userId + references: + - dataset: hubspot_connector_example + field: users.id + direction: from + +``` diff --git a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py index a42851373..5055f4424 100644 --- a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py @@ -230,14 +230,14 @@ def test_get_connection_secret_schema_hubspot( "description": "Hubspot secrets schema", "type": "object", "properties": { - "hapikey": {"title": "Hapikey", "type": "string"}, + "private_app_token": {"title": "Private App Token", "type": "string"}, "domain": { "title": "Domain", "default": "api.hubapi.com", "type": "string", }, }, - "required": ["hapikey"], + "required": ["private_app_token"], "additionalProperties": False, } diff --git a/tests/ops/fixtures/saas/hubspot_fixtures.py b/tests/ops/fixtures/saas/hubspot_fixtures.py index a8a95d0bb..53e82c72d 100644 --- a/tests/ops/fixtures/saas/hubspot_fixtures.py +++ b/tests/ops/fixtures/saas/hubspot_fixtures.py @@ -1,8 +1,8 @@ -import json from typing import Any, Dict, Generator import pydash import pytest +import requests from fideslib.cryptography import cryptographic_util from sqlalchemy.orm import Session @@ -12,10 +12,7 @@ ConnectionType, ) from fidesops.ops.models.datasetconfig import DatasetConfig -from fidesops.ops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams -from fidesops.ops.service.connectors import SaaSConnector from fidesops.ops.util.saas_util import ( - format_body, load_config_with_replacement, load_dataset_with_replacement, ) @@ -24,14 +21,13 @@ secrets = get_secrets("hubspot") -HUBSPOT_FIRSTNAME = "SomeoneFirstname" - @pytest.fixture(scope="session") def hubspot_secrets(saas_config): return { "domain": pydash.get(saas_config, "hubspot.domain") or secrets["domain"], - "hapikey": pydash.get(saas_config, "hubspot.hapikey") or secrets["hapikey"], + "private_app_token": pydash.get(saas_config, "hubspot.private_app_token") + or secrets["private_app_token"], } @@ -110,63 +106,177 @@ def dataset_config_hubspot( dataset.delete(db=db) -@pytest.fixture(scope="function") -def hubspot_erasure_data( - connection_config_hubspot, hubspot_erasure_identity_email -) -> Generator: - """ - Gets the current value of the resource and restores it after the test is complete. - Used for erasure tests. - """ +class HubspotTestClient: - connector = SaaSConnector(connection_config_hubspot) + headers: object = {} + base_url: str = "" - body = json.dumps( - { + def __init__(self, connection_config_hubspot: ConnectionConfig): + hubspot_secrets = connection_config_hubspot.secrets + self.headers = { + "Authorization": f"Bearer {hubspot_secrets['private_app_token']}", + } + self.base_url = f"https://{hubspot_secrets['domain']}" + + def get_user(self, user_id: str) -> requests.Response: + user_response: requests.Response = requests.get( + url=f"{self.base_url}/settings/v3/users/{user_id}", headers=self.headers + ) + return user_response + + def get_contact(self, contact_id: str) -> requests.Response: + contact_response: requests.Response = requests.get( + url=f"{self.base_url}/crm/v3/objects/contacts/{contact_id}", + headers=self.headers, + ) + return contact_response + + def get_contact_by_email(self, email: str) -> requests.Response: + body = { + "filterGroups": [ + { + "filters": [ + { + "value": email, + "propertyName": "email", + "operator": "EQ", + } + ] + } + ] + } + contact_search_response: requests.Response = requests.post( + url=f"{self.base_url}/crm/v3/objects/contacts/search", + json=body, + headers=self.headers, + ) + return contact_search_response + + def create_contact(self, email: str) -> requests.Response: + contacts_request_body = { "properties": { "company": "test company", - "email": hubspot_erasure_identity_email, - "firstname": HUBSPOT_FIRSTNAME, + "email": email, + "firstname": "SomeoneFirstname", "lastname": "SomeoneLastname", "phone": "(123) 123-1234", "website": "someone.net", } } - ) + contact_response: requests.Response = requests.post( + url=f"{self.base_url}/crm/v3/objects/contacts", + headers=self.headers, + json=contacts_request_body, + ) + return contact_response + + def create_user(self, email: str) -> requests.Response: + users_request_body = { + "email": email, + } + user_response: requests.Response = requests.post( + url=f"{self.base_url}/settings/v3/users/", + headers=self.headers, + json=users_request_body, + ) + return user_response + + def delete_contact(self, contact_id: str) -> requests.Response: + contact_response: requests.Response = requests.delete( + url=f"{self.base_url}/crm/v3/objects/contacts/{contact_id}", + headers=self.headers, + ) + return contact_response + + def get_email_subscriptions(self, email: str) -> requests.Response: + email_subscriptions: requests.Response = requests.get( + url=f"{self.base_url}/communication-preferences/v3/status/email/{email}", + headers=self.headers, + ) + return email_subscriptions + + +@pytest.fixture(scope="function") +def hubspot_test_client( + connection_config_hubspot: HubspotTestClient, +) -> Generator: + test_client = HubspotTestClient(connection_config_hubspot=connection_config_hubspot) + yield test_client + + +def _contact_exists( + contact_id: str, email: str, hubspot_test_client: HubspotTestClient +) -> Any: + """ + Confirm whether contact exists. We check the crm search endpoint as this is + what our connectors use. + """ + contact_response = hubspot_test_client.get_contact(contact_id=contact_id) + contact_search_response = hubspot_test_client.get_contact_by_email(email=email) + + if ( + not contact_response.status_code == 404 + and contact_search_response.json()["results"] + ): + contact_body = contact_response.json() + return contact_body + + +def user_exists(user_id: str, hubspot_test_client: HubspotTestClient) -> Any: + """ + Confirm whether user exists + """ + user_response: requests.Response = hubspot_test_client.get_user(user_id=user_id) + if not user_response.status_code == 404: + user_body = user_response.json() + return user_body - updated_headers, formatted_body = format_body({}, body) +@pytest.fixture(scope="function") +def hubspot_erasure_data( + hubspot_test_client: HubspotTestClient, + hubspot_erasure_identity_email: str, +) -> Generator: + """ + Gets the current value of the resource and restores it after the test is complete. + Used for erasure tests. + """ # create contact - contacts_request: SaaSRequestParams = SaaSRequestParams( - method=HTTPMethod.POST, - path=f"/crm/v3/objects/contacts", - headers=updated_headers, - body=formatted_body, + contacts_response = hubspot_test_client.create_contact( + email=hubspot_erasure_identity_email ) - contacts_response = connector.create_client().send(contacts_request) contacts_body = contacts_response.json() contact_id = contacts_body["id"] - # no need to subscribe contact, since creating a contact auto-subscribes them + # create user + users_response = hubspot_test_client.create_user( + email=hubspot_erasure_identity_email + ) + users_body = users_response.json() + user_id = users_body["id"] + # no need to subscribe contact, since creating a contact auto-subscribes them # Allows contact to be propagated in Hubspot before calling access / erasure requests error_message = ( f"Contact with contact id {contact_id} could not be added to Hubspot" ) poll_for_existence( _contact_exists, - (hubspot_erasure_identity_email, connector), + (contact_id, hubspot_erasure_identity_email, hubspot_test_client), error_message=error_message, ) - yield contact_id + error_message = f"User with user id {user_id} could not be added to Hubspot" + poll_for_existence( + user_exists, + (user_id, hubspot_test_client), + error_message=error_message, + ) + + yield contact_id, user_id # delete contact - delete_request: SaaSRequestParams = SaaSRequestParams( - method=HTTPMethod.DELETE, - path=f"/crm/v3/objects/contacts/{contact_id}", - ) - connector.create_client().send(delete_request) + hubspot_test_client.delete_contact(contact_id=contact_id) # verify contact is deleted error_message = ( @@ -174,46 +284,7 @@ def hubspot_erasure_data( ) poll_for_existence( _contact_exists, - (hubspot_erasure_identity_email, connector), + (contact_id, hubspot_erasure_identity_email, hubspot_test_client), error_message=error_message, existence_desired=False, ) - - -def _contact_exists( - hubspot_erasure_identity_email: str, connector: SaaSConnector -) -> Any: - """ - Confirm whether contact exists by calling search api and comparing firstname str. - """ - - body = json.dumps( - { - "filterGroups": [ - { - "filters": [ - { - "value": hubspot_erasure_identity_email, - "propertyName": "email", - "operator": "EQ", - } - ] - } - ] - } - ) - - updated_headers, formatted_body = format_body({}, body) - contact_request: SaaSRequestParams = SaaSRequestParams( - method=HTTPMethod.POST, - path="/crm/v3/objects/contacts/search", - headers=updated_headers, - body=formatted_body, - ) - contact_response = connector.create_client().send(contact_request) - contact_body = contact_response.json() - if ( - contact_body["results"] - and contact_body["results"][0]["properties"]["firstname"] == HUBSPOT_FIRSTNAME - ): - return contact_body diff --git a/tests/ops/integration_tests/saas/test_hubspot_task.py b/tests/ops/integration_tests/saas/test_hubspot_task.py index eb4f2e8bb..53a01b44a 100644 --- a/tests/ops/integration_tests/saas/test_hubspot_task.py +++ b/tests/ops/integration_tests/saas/test_hubspot_task.py @@ -1,18 +1,24 @@ -import json import random import pytest +from fidesops.ops.core.config import config from fidesops.ops.graph.graph import DatasetGraph -from fidesops.ops.models.privacy_request import ExecutionLog, PrivacyRequest +from fidesops.ops.models.privacy_request import PrivacyRequest from fidesops.ops.schemas.redis_cache import PrivacyRequestIdentity -from fidesops.ops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams -from fidesops.ops.service.connectors import SaaSConnector +from fidesops.ops.service.connectors import get_connector from fidesops.ops.task import graph_task from fidesops.ops.task.filter_results import filter_data_categories from fidesops.ops.task.graph_task import get_cached_data_for_erasures -from fidesops.ops.util.saas_util import format_body -from tests.ops.graph.graph_test_util import assert_rows_match, records_matching_fields +from tests.ops.fixtures.saas.hubspot_fixtures import HubspotTestClient, user_exists +from tests.ops.graph.graph_test_util import assert_rows_match +from tests.ops.test_helpers.saas_test_utils import poll_for_existence + + +@pytest.mark.integration_saas +@pytest.mark.integration_hubspot +def test_hubspot_connection_test(connection_config_hubspot) -> None: + get_connector(connection_config_hubspot).test_connection() @pytest.mark.integration_saas @@ -58,6 +64,16 @@ def test_saas_access_request_task( min_size=1, keys=["recipient", "subscriptionStatuses"], ) + assert_rows_match( + v[f"{dataset_name}:users"], + min_size=1, + keys=["id", "email"], + ) + assert_rows_match( + v[f"{dataset_name}:owners"], + min_size=1, + keys=["id", "email", "lastName", "updatedAt", "firstName", "userId"], + ) target_categories = {"user"} filtered_results = filter_data_categories( @@ -69,6 +85,8 @@ def test_saas_access_request_task( assert set(filtered_results.keys()) == { f"{dataset_name}:contacts", f"{dataset_name}:subscription_preferences", + f"{dataset_name}:users", + f"{dataset_name}:owners", } assert set(filtered_results[f"{dataset_name}:contacts"][0].keys()) == { "id", @@ -91,41 +109,20 @@ def test_saas_access_request_task( filtered_results[f"{dataset_name}:subscription_preferences"][0]["recipient"] == hubspot_identity_email ) - - logs = ( - ExecutionLog.query(db=db) - .filter(ExecutionLog.privacy_request_id == privacy_request.id) - .all() - ) - - logs = [log.__dict__ for log in logs] + assert set(filtered_results[f"{dataset_name}:users"][0].keys()) == {"email", "id"} assert ( - len( - records_matching_fields( - logs, dataset_name=dataset_name, collection_name="contacts" - ) - ) - > 0 - ) - assert ( - len( - records_matching_fields( - logs, - dataset_name=dataset_name, - collection_name="owners", - ) - ) - > 0 + filtered_results[f"{dataset_name}:users"][0]["email"] == hubspot_identity_email ) + assert set(filtered_results[f"{dataset_name}:owners"][0].keys()) == { + "email", + "id", + "userId", + "updatedAt", + "firstName", + "lastName", + } assert ( - len( - records_matching_fields( - logs, - dataset_name=dataset_name, - collection_name="subscription_preferences", - ) - ) - > 0 + filtered_results[f"{dataset_name}:owners"][0]["email"] == hubspot_identity_email ) @@ -139,8 +136,10 @@ def test_saas_erasure_request_task( dataset_config_hubspot, hubspot_erasure_identity_email, hubspot_erasure_data, + hubspot_test_client: HubspotTestClient, ) -> None: """Full erasure request based on the Hubspot SaaS config""" + contact_id, user_id = hubspot_erasure_data privacy_request = PrivacyRequest( id=f"test_hubspot_erasure_request_task_{random.randint(0, 1000)}" ) @@ -168,6 +167,8 @@ def test_saas_erasure_request_task( keys=["recipient", "subscriptionStatuses"], ) + temp_masking = config.execution.masking_strict + config.execution.masking_strict = False # Allow delete erasure = graph_task.run_erasure( privacy_request, erasure_policy_string_rewrite, @@ -177,50 +178,33 @@ def test_saas_erasure_request_task( get_cached_data_for_erasures(privacy_request.id), db, ) + config.execution.masking_strict = temp_masking - # Masking request only issued to "contacts" and "subscription_preferences" endpoints + # Masking request only issued to "contacts", "subscription_preferences", and "users" endpoints assert erasure == { "hubspot_instance:contacts": 1, "hubspot_instance:owners": 0, "hubspot_instance:subscription_preferences": 1, + "hubspot_instance:users": 1, } - connector = SaaSConnector(connection_config_hubspot) - - body = json.dumps( - { - "filterGroups": [ - { - "filters": [ - { - "value": hubspot_erasure_identity_email, - "propertyName": "email", - "operator": "EQ", - } - ] - } - ] - } - ) - - updated_headers, formatted_body = format_body({}, body) - # Verify the user has been assigned to None - contact_request: SaaSRequestParams = SaaSRequestParams( - method=HTTPMethod.POST, - path="/crm/v3/objects/contacts/search", - headers=updated_headers, - body=formatted_body, - ) - contact_response = connector.create_client().send(contact_request) + contact_response = hubspot_test_client.get_contact(contact_id=contact_id) contact_body = contact_response.json() - assert contact_body["results"][0]["properties"]["firstname"] == "MASKED" + assert contact_body["properties"]["firstname"] == "MASKED" # verify user is unsubscribed - subscription_request: SaaSRequestParams = SaaSRequestParams( - method=HTTPMethod.GET, - path=f"/communication-preferences/v3/status/email/{hubspot_erasure_identity_email}", + email_subscription_response = hubspot_test_client.get_email_subscriptions( + email=hubspot_erasure_identity_email ) - subscription_response = connector.create_client().send(subscription_request) - subscription_body = subscription_response.json() + subscription_body = email_subscription_response.json() assert subscription_body["subscriptionStatuses"][0]["status"] == "NOT_SUBSCRIBED" + + # verify user is deleted + error_message = f"User with user id {user_id} could not be deleted from Hubspot" + poll_for_existence( + user_exists, + (user_id, hubspot_test_client), + error_message=error_message, + existence_desired=False, + ) diff --git a/tests/ops/service/privacy_request/request_runner_service_test.py b/tests/ops/service/privacy_request/request_runner_service_test.py index 0d7e15f73..b18ae749a 100644 --- a/tests/ops/service/privacy_request/request_runner_service_test.py +++ b/tests/ops/service/privacy_request/request_runner_service_test.py @@ -594,7 +594,7 @@ def test_create_and_process_access_request_saas_hubspot( task_timeout=PRIVACY_REQUEST_TASK_TIMEOUT_EXTERNAL, ) results = pr.get_results() - assert len(results.keys()) == 3 + assert len(results.keys()) == 4 for key in results.keys(): assert results[key] is not None