diff --git a/CHANGELOG.md b/CHANGELOG.md index cb28e7fcc..e316f9fa0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ The types of changes are: * Mapping Vault environment variables in docker-compose.yml [#1275](https://github.com/ethyca/fidesops/pull/1275) * Foundations for a new "manual_webhook" connector type [#1267](https://github.com/ethyca/fidesops/pull/1267) * Data seeding for Datadog access tests [#1269](https://github.com/ethyca/fidesops/pull/1269) +* Added support for one-to-many relationships for param_values in SaaS configs [#1253](https://github.com/ethyca/fidesops/pull/1253) ### Docs diff --git a/data/saas/config/saas_example_config.yml b/data/saas/config/saas_example_config.yml index c71cb3a16..183526926 100644 --- a/data/saas/config/saas_example_config.yml +++ b/data/saas/config/saas_example_config.yml @@ -12,6 +12,9 @@ saas_config: - name: api_key - name: api_version - name: page_size + - name: account_types + options: [checking, savings, investment] + multiselect: True client_config: protocol: https @@ -38,10 +41,8 @@ saas_config: - dataset: saas_connector_example field: conversations.id direction: from + data_path: conversation_messages postprocessors: - - strategy: unwrap - configuration: - data_path: conversation_messages - strategy: filter configuration: field: from_email @@ -61,10 +62,7 @@ saas_config: param_values: - name: placeholder identity: email - postprocessors: - - strategy: unwrap - configuration: - data_path: conversations + data_path: conversations - name: member requests: read: @@ -76,10 +74,7 @@ saas_config: param_values: - name: email identity: email - postprocessors: - - strategy: unwrap - configuration: - data_path: exact_matches.members + data_path: exact_matches.members update: method: PUT path: /3.0/lists//members/ @@ -201,3 +196,39 @@ saas_config: { "unique_id": "" } + - name: accounts + requests: + read: + method: GET + path: /v1//account + grouped_inputs: [customer_id, customer_name] + query_params: + - name: account_type + value: + - name: customer_name + value: + param_values: + - name: account_type + connector_param: account_types + - name: customer_id + references: + - dataset: saas_connector_example + field: customer.id + direction: from + - name: customer_name + references: + - dataset: saas_connector_example + field: customer.name + direction: from + - name: mailing_lists + requests: + read: + method: GET + path: /v1/mailing_list/ + param_values: + - name: list_id + references: + - dataset: saas_connector_example + field: users.list_ids + direction: from + unpack: True diff --git a/data/saas/config/stripe_config.yml b/data/saas/config/stripe_config.yml index 2a8b00d48..676ff1d93 100644 --- a/data/saas/config/stripe_config.yml +++ b/data/saas/config/stripe_config.yml @@ -85,8 +85,6 @@ saas_config: query_params: - name: charge value: - - name: payment_intent - value: - name: limit value: param_values: @@ -95,11 +93,6 @@ saas_config: - dataset: field: charge.id direction: from - - name: payment_intent_id - references: - - dataset: - field: payment_intent.id - direction: from - name: limit connector_param: page_size data_path: data diff --git a/data/saas/dataset/saas_example_dataset.yml b/data/saas/dataset/saas_example_dataset.yml index 1974cb279..1a32ae7f9 100644 --- a/data/saas/dataset/saas_example_dataset.yml +++ b/data/saas/dataset/saas_example_dataset.yml @@ -180,6 +180,9 @@ dataset: data_categories: [user.device.ip_address] - name: email data_categories: [user.contact.email] + - name: list_ids + fidesops_meta: + data_type: string[] - name: customer fields: - name: id @@ -207,3 +210,17 @@ dataset: data_categories: [system.operations] fidesops_meta: data_type: string + - name: accounts + fields: + - name: name + data_categories: [system.operations] + fidesops_meta: + data_type: string + - name: mailing_lists + fields: + - name: list_id + fidesops_meta: + data_type: string + - name: mailing_list_name + fidesops_meta: + data_type: string diff --git a/src/fidesops/ops/schemas/connection_configuration/connection_secrets_saas.py b/src/fidesops/ops/schemas/connection_configuration/connection_secrets_saas.py index 0780a7288..881f2ff18 100644 --- a/src/fidesops/ops/schemas/connection_configuration/connection_secrets_saas.py +++ b/src/fidesops/ops/schemas/connection_configuration/connection_secrets_saas.py @@ -1,7 +1,7 @@ import abc from typing import Any, Dict, Type -from pydantic import BaseModel, Extra, create_model, root_validator +from pydantic import BaseModel, Extra, PrivateAttr, create_model, root_validator from fidesops.ops.schemas.saas.saas_config import SaaSConfig @@ -15,10 +15,11 @@ class SaaSSchema(BaseModel, abc.ABC): @root_validator @classmethod def required_components_supplied( # type: ignore - cls: BaseModel, values: Dict[str, Any] + cls, values: Dict[str, Any] ) -> Dict[str, Any]: """Validate that the minimum required components have been supplied.""" + # check required components are present required_components = [ name for name, attributes in cls.__fields__.items() if attributes.required ] @@ -30,8 +31,35 @@ def required_components_supplied( # type: ignore f"{cls.__name__} must be supplied all of: [{', '.join(required_components)}]." # type: ignore ) + # check the types and values are consistent with the option and multivalue fields + for name, value in values.items(): + connector_param = cls.get_connector_param(name) + options = connector_param.get("options") + multiselect = connector_param.get("multiselect") + + if options: + if isinstance(value, str): + if value not in options: + raise ValueError( + f"'{name}' must be one of [{', '.join(options)}]" + ) + elif isinstance(value, list): + if not multiselect: + raise ValueError( + "f'{name}' must be a single value when multiselect is not enabled, not a list" + ) + invalid_options = [entry for entry in value if entry not in options] + if invalid_options: + raise ValueError( + f"[{', '.join(invalid_options)}] are not valid options, '{name}' must be a list of values from [{', '.join(options)}]" + ) + return values + @classmethod + def get_connector_param(cls, name: str) -> Dict[str, Any]: + return cls.__private_attributes__.get("_connector_params").default.get(name) # type: ignore + class Config: """Only permit selected secret fields to be stored.""" @@ -48,21 +76,34 @@ class SaaSSchemaFactory: def __init__(self, saas_config: SaaSConfig): self.saas_config = saas_config - # Pydantic uses the shorthand of (str, ...) to denote a required field of type str + # Pydantic uses the shorthand of (type, ...) to denote a required field of the given type def get_saas_schema(self) -> Type[SaaSSchema]: """Returns the schema for the current configuration""" field_definitions: Dict[str, Any] = {} for connector_param in self.saas_config.connector_params: + param_type = list if connector_param.multiselect else str field_definitions[connector_param.name] = ( connector_param.default_value if connector_param.default_value - else (str, ...) + else (param_type, ...) ) SaaSSchema.__doc__ = f"{str(self.saas_config.type).capitalize()} secrets schema" # Dynamically override the docstring to create a description + + # set the connector_params as a private attribute on the schema class + # so they can be accessible in the 'required_components_supplied' validator model: Type[SaaSSchema] = create_model( f"{self.saas_config.type}_schema", **field_definitions, __base__=SaaSSchema, + _connector_params=PrivateAttr( + { + connector_param.name: { + "options": connector_param.options, + "multiselect": connector_param.multiselect, + } + for connector_param in self.saas_config.connector_params + } + ), ) return model diff --git a/src/fidesops/ops/schemas/saas/saas_config.py b/src/fidesops/ops/schemas/saas/saas_config.py index d59d80320..3e00e384a 100644 --- a/src/fidesops/ops/schemas/saas/saas_config.py +++ b/src/fidesops/ops/schemas/saas/saas_config.py @@ -26,6 +26,7 @@ class ParamValue(BaseModel): identity: Optional[str] references: Optional[List[FidesopsDatasetReference]] connector_param: Optional[str] + unpack: Optional[bool] = False @validator("references") def check_reference_direction( @@ -201,9 +202,46 @@ class ConnectorParam(BaseModel): """Used to define the required parameters for the connector (user and constants)""" name: str - default_value: Optional[str] + options: Optional[List[str]] # list of possible values for the connector param + default_value: Optional[Union[str, List[str]]] + multiselect: Optional[bool] = False description: Optional[str] + @root_validator + def validate_connector_param(cls, values: Dict[str, Any]) -> Dict[str, Any]: + """Verify the default_value is one of the values specified in the options list""" + + name = values.get("name") + options: Optional[List[str]] = values.get("options") + default_value: Optional[Union[str, List[str]]] = values.get("default_value") + multiselect: Optional[bool] = values.get("multiselect") + + if options: + if isinstance(default_value, str) and default_value not in options: + raise ValueError( + f"'{default_value}' is not a valid option, default_value must be a value from [{', '.join(options)}]" + ) + if isinstance(default_value, list): + if not multiselect: + raise ValueError( + f"The default_value for the {name} connector_param must be a single value when multiselect is not enabled, not a list" + ) + + invalid_options = [ + value for value in default_value if value not in options + ] + if invalid_options: + raise ValueError( + f"[{', '.join(invalid_options)}] are not valid options, default_value must be a list of values from [{', '.join(options)}]" + ) + + if multiselect and not options: + raise ValueError( + f"The 'multiselect' field in the {name} connector_param must be accompanied by an 'options' field containing a list of values." + ) + + return values + class SaaSType(Enum): """ diff --git a/src/fidesops/ops/service/connectors/saas_query_config.py b/src/fidesops/ops/service/connectors/saas_query_config.py index b119645d8..ee369fd2c 100644 --- a/src/fidesops/ops/service/connectors/saas_query_config.py +++ b/src/fidesops/ops/service/connectors/saas_query_config.py @@ -2,6 +2,7 @@ import json import logging +from itertools import product from typing import Any, Dict, List, Optional, TypeVar import pydash @@ -17,7 +18,13 @@ from fidesops.ops.service.connectors.query_config import QueryConfig from fidesops.ops.util import saas_util from fidesops.ops.util.collection_util import Row, merge_dicts -from fidesops.ops.util.saas_util import FIDESOPS_GROUPED_INPUTS, unflatten_dict +from fidesops.ops.util.saas_util import ( + ALL_OBJECT_FIELDS, + FIDESOPS_GROUPED_INPUTS, + MASKED_OBJECT_FIELDS, + PRIVACY_REQUEST_ID, + unflatten_dict, +) logger = logging.getLogger(__name__) @@ -106,28 +113,91 @@ def get_masking_request(self) -> Optional[SaaSRequest]: def generate_requests( self, input_data: Dict[str, List[Any]], policy: Optional[Policy] ) -> List[SaaSRequestParams]: - """Takes the input_data and uses it to generate a list of SaaS request params""" + """ + Takes the identity and reference values from input_data and combines them + with the connector_param values in use by the current request to generate + a list of request params. + """ - filtered_data = self.node.typed_filtered_values(input_data) + current_request: Optional[SaaSRequest] = self.get_request_by_action("read") + if not current_request: + raise FidesopsException( + f"The 'read' action is not defined for the '{self.collection_name}' " + f"endpoint in {self.node.node.dataset.connection_key}" + ) request_params = [] - # Build SaaS requests for fields that are independent of each other - for string_path, reference_values in filtered_data.items(): - for value in reference_values: + filtered_secrets = self._filtered_secrets(current_request) + grouped_inputs_list = input_data.pop(FIDESOPS_GROUPED_INPUTS, None) + + # unpack the inputs + # list_ids: [[1,2,3]] -> list_ids: [1,2,3] + for param_value in current_request.param_values or []: + if param_value.unpack: + value = param_value.name + input_data[value] = pydash.flatten(input_data.get(value)) + + # we want to preserve the grouped_input relationships so we take each + # individual group and generate the product with the ungrouped inputs + for grouped_inputs in grouped_inputs_list or [{}]: + param_value_maps = self._generate_product_list( + input_data, filtered_secrets, grouped_inputs + ) + for param_value_map in param_value_maps: request_params.append( - self.generate_query({string_path: [value]}, policy) + self.generate_query( + {name: [value] for name, value in param_value_map.items()}, + policy, + ) ) - # Build SaaS requests for fields that are dependent on each other - grouped_input_data: List[Dict[str, Any]] = input_data.get( - FIDESOPS_GROUPED_INPUTS, [] - ) - for dependent_data in grouped_input_data: - request_params.append(self.generate_query(dependent_data, policy)) - return request_params + def _filtered_secrets(self, current_request: SaaSRequest) -> Dict[str, Any]: + """Return a filtered map of secrets used by the request""" + + param_names = [ + param_value.connector_param + for param_value in current_request.param_values or [] + ] + return { + name: value for name, value in self.secrets.items() if name in param_names + } + + @staticmethod + def _generate_product_list(*args: Dict[str, Any]) -> List[Dict[str, Any]]: + """ + Accepts a variable number of dicts and produces the product of the values from all the dicts. + + Example: + + `_generate_product_list({ "first": ["a", "b"] }, { "second": ["1", "2", "3"] })` + + Returns: + ``` + [ + { "first": "a", "second": "1" } + { "first": "a", "second": "2" } + { "first": "a", "second": "3" } + { "first": "b", "second": "1" } + { "first": "b", "second": "2" } + { "first": "b", "second": "3" } + ] + ``` + """ + + merged_dicts = merge_dicts(*args) + return [ + dict(zip(merged_dicts.keys(), values)) + for values in product( + *( + value if isinstance(value, list) else [value] + for value in merged_dicts.values() + ) + ) + ] + def generate_query( self, input_data: Dict[str, List[Any]], policy: Optional[Policy] ) -> SaaSRequestParams: @@ -137,7 +207,7 @@ def generate_query( query statement (select statement, where clause, limit, offset, etc.) """ - current_request: SaaSRequest | None = self.get_request_by_action("read") + current_request: Optional[SaaSRequest] = self.get_request_by_action("read") if not current_request: raise FidesopsException( f"The 'read' action is not defined for the '{self.collection_name}' " @@ -149,18 +219,16 @@ def generate_query( param_values: Dict[str, Any] = {} for param_value in current_request.param_values or []: if param_value.references or param_value.identity: - # TODO: how to handle missing reference or identity values in a way - # in a way that is obvious based on configuration input_list = input_data.get(param_value.name) if input_list: param_values[param_value.name] = input_list[0] elif param_value.connector_param: param_values[param_value.name] = pydash.get( - self.secrets, param_value.connector_param - ) + input_data, param_value.connector_param + )[0] if self.privacy_request: - param_values["privacy_request_id"] = self.privacy_request.id + param_values[PRIVACY_REQUEST_ID] = self.privacy_request.id # map param values to placeholders in path, headers, and query params saas_request_params: SaaSRequestParams = saas_util.map_param_values( @@ -226,7 +294,7 @@ def generate_update_param_values( # pylint: disable=R0914 ) if self.privacy_request: - param_values["privacy_request_id"] = self.privacy_request.id + param_values[PRIVACY_REQUEST_ID] = self.privacy_request.id # remove any row values for fields marked as read-only, these will be omitted from all update maps for field_path, field in self.field_map().items(): @@ -247,8 +315,8 @@ def generate_update_param_values( # pylint: disable=R0914 merge_dicts(all_value_map, update_value_map) ) - param_values["masked_object_fields"] = masked_object - param_values["all_object_fields"] = complete_object + param_values[MASKED_OBJECT_FIELDS] = masked_object + param_values[ALL_OBJECT_FIELDS] = complete_object return param_values @@ -261,12 +329,12 @@ def generate_update_request_params( """ # removes outer {} wrapper from body for greater flexibility in custom body config - param_values["masked_object_fields"] = json.dumps( - param_values["masked_object_fields"] - )[1:-1] - param_values["all_object_fields"] = json.dumps( - param_values["all_object_fields"] + param_values[MASKED_OBJECT_FIELDS] = json.dumps( + param_values[MASKED_OBJECT_FIELDS] )[1:-1] + param_values[ALL_OBJECT_FIELDS] = json.dumps(param_values[ALL_OBJECT_FIELDS])[ + 1:-1 + ] # map param values to placeholders in path, headers, and query params saas_request_params: SaaSRequestParams = saas_util.map_param_values( diff --git a/src/fidesops/ops/util/saas_util.py b/src/fidesops/ops/util/saas_util.py index be4be17aa..abf256cf2 100644 --- a/src/fidesops/ops/util/saas_util.py +++ b/src/fidesops/ops/util/saas_util.py @@ -19,6 +19,9 @@ logger = logging.getLogger(__name__) FIDESOPS_GROUPED_INPUTS = "fidesops_grouped_inputs" +PRIVACY_REQUEST_ID = "privacy_request_id" +MASKED_OBJECT_FIELDS = "masked_object_fields" +ALL_OBJECT_FIELDS = "all_object_fields" def load_yaml_as_string(filename: str) -> str: diff --git a/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py b/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py index 193a01928..17e20da06 100644 --- a/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py @@ -244,7 +244,7 @@ def test_patch_saas_config_update( ) saas_config = connection_config.saas_config assert saas_config is not None - assert len(saas_config["endpoints"]) == 7 + assert len(saas_config["endpoints"]) == 9 def get_saas_config_url(connection_config: Optional[ConnectionConfig] = None) -> str: @@ -318,7 +318,7 @@ def test_get_saas_config( response_body["fides_key"] == saas_example_connection_config.get_saas_config().fides_key ) - assert len(response_body["endpoints"]) == 8 + assert len(response_body["endpoints"]) == 10 assert response_body["type"] == "custom" diff --git a/tests/ops/fixtures/saas/zendesk_fixtures.py b/tests/ops/fixtures/saas/zendesk_fixtures.py index db13496c1..2eb6bb79f 100644 --- a/tests/ops/fixtures/saas/zendesk_fixtures.py +++ b/tests/ops/fixtures/saas/zendesk_fixtures.py @@ -1,3 +1,4 @@ +import time from typing import Any, Dict, Generator import pydash @@ -27,6 +28,8 @@ def zendesk_secrets(saas_config): "domain": pydash.get(saas_config, "zendesk.domain") or secrets["domain"], "username": pydash.get(saas_config, "zendesk.username") or secrets["username"], "api_key": pydash.get(saas_config, "zendesk.api_key") or secrets["api_key"], + "page_size": pydash.get(saas_config, "zendesk.page_size") + or secrets["page_size"], } @@ -107,6 +110,8 @@ def zendesk_create_erasure_data( zendesk_connection_config: ConnectionConfig, zendesk_erasure_identity_email: str ) -> None: + time.sleep(60) + zendesk_secrets = zendesk_connection_config.secrets auth = zendesk_secrets["username"], zendesk_secrets["api_key"] base_url = f"https://{zendesk_secrets['domain']}" diff --git a/tests/ops/fixtures/saas_example_fixtures.py b/tests/ops/fixtures/saas_example_fixtures.py index 0eea262c6..e2b76dac4 100644 --- a/tests/ops/fixtures/saas_example_fixtures.py +++ b/tests/ops/fixtures/saas_example_fixtures.py @@ -27,7 +27,8 @@ def saas_example_secrets(): "username": "username", "api_key": "api_key", "api_version": "2.0", - "page_size": "page_size", + "account_types": ["checking"], + "page_size": "10", } diff --git a/tests/ops/integration_tests/saas/test_zendesk_task.py b/tests/ops/integration_tests/saas/test_zendesk_task.py index 16c8328d2..fb72eb1f9 100644 --- a/tests/ops/integration_tests/saas/test_zendesk_task.py +++ b/tests/ops/integration_tests/saas/test_zendesk_task.py @@ -137,7 +137,6 @@ async def test_zendesk_access_request_task( "custom_fields", "satisfaction_rating", "sharing_agreement_ids", - "fields", "followup_ids", "brand_id", "allow_channelback", @@ -290,7 +289,6 @@ async def test_zendesk_erasure_request_task( "custom_fields", "satisfaction_rating", "sharing_agreement_ids", - "fields", "followup_ids", "brand_id", "allow_channelback", diff --git a/tests/ops/models/test_saasconfig.py b/tests/ops/models/test_saasconfig.py index f4e25f1ed..6fe8b0017 100644 --- a/tests/ops/models/test_saasconfig.py +++ b/tests/ops/models/test_saasconfig.py @@ -5,7 +5,12 @@ from fidesops.ops.graph.config import CollectionAddress, FieldAddress from fidesops.ops.schemas.dataset import FidesopsDatasetReference -from fidesops.ops.schemas.saas.saas_config import ParamValue, SaaSConfig, SaaSRequest +from fidesops.ops.schemas.saas.saas_config import ( + ConnectorParam, + ParamValue, + SaaSConfig, + SaaSRequest, +) @pytest.mark.unit_saas @@ -137,3 +142,65 @@ def test_saas_config_ignore_errors_param(saas_example_config: Dict[str, Dict]): member_endpoint = next(end for end in saas_config.endpoints if end.name == "member") # Not specified on member read endpoint - defaults to False assert not member_endpoint.requests["read"].ignore_errors + + +@pytest.mark.unit_saas +class TestConnectorParam: + def test_name_only(self): + ConnectorParam(name="account_type") + + def test_single_default_value(self): + ConnectorParam(name="account_type", default_value="checking") + + def test_list_default_values(self): + ConnectorParam(name="account_types", default_value=["checking", "savings"]) + + def test_missing_name(self): + with pytest.raises(ValidationError) as exc: + ConnectorParam(default_value="checking") + assert "field required" in str(exc.value) + + def test_default_value_not_in_options(self): + with pytest.raises(ValidationError) as exc: + ConnectorParam( + name="account_types", + default_value="roth", + options=["checking", "savings"], + ) + assert ( + "'roth' is not a valid option, default_value must be a value from [checking, savings]" + in str(exc.value) + ) + + def test_default_values_not_in_options(self): + with pytest.raises(ValidationError) as exc: + ConnectorParam( + name="account_types", + default_value=["roth", "401k"], + options=["checking", "savings"], + multiselect=True, + ) + assert ( + "[roth, 401k] are not valid options, default_value must be a list of values from [checking, savings]" + in str(exc.value) + ) + + def test_multiselect_without_options(self): + with pytest.raises(ValidationError) as exc: + ConnectorParam(name="account_types", multiselect=True) + assert ( + "The 'multiselect' field in the account_types connector_param must be accompanied by an 'options' field containing a list of values." + in str(exc.value) + ) + + def test_default_value_list_without_multiselect(self): + with pytest.raises(ValidationError) as exc: + ConnectorParam( + name="account_types", + default_value=["checking", "savings"], + options=["checking", "savings"], + ) + assert ( + "The default_value for the account_types connector_param must be a single value when multiselect is not enabled, not a list" + in str(exc.value) + ) diff --git a/tests/ops/schemas/connection_configuration/test_connection_secrets_saas.py b/tests/ops/schemas/connection_configuration/test_connection_secrets_saas.py index 45a2a1de9..6e26da45b 100644 --- a/tests/ops/schemas/connection_configuration/test_connection_secrets_saas.py +++ b/tests/ops/schemas/connection_configuration/test_connection_secrets_saas.py @@ -7,7 +7,7 @@ SaaSSchema, SaaSSchemaFactory, ) -from fidesops.ops.schemas.saas.saas_config import SaaSConfig +from fidesops.ops.schemas.saas.saas_config import ConnectorParam, SaaSConfig @pytest.mark.unit_saas @@ -72,3 +72,42 @@ def test_default_value_fields( del saas_example_secrets["domain"] config = saas_example_secrets schema.parse_obj(config) + + def test_value_in_options(self, saas_config: SaaSConfig): + saas_config.connector_params = [ + ConnectorParam(name="account_type", options=["checking", "savings"]) + ] + schema = SaaSSchemaFactory(saas_config).get_saas_schema() + schema.parse_obj({"account_type": "checking"}) + + def test_value_in_options_with_multiselect(self, saas_config: SaaSConfig): + saas_config.connector_params = [ + ConnectorParam( + name="account_type", options=["checking", "savings"], multiselect=True + ) + ] + schema = SaaSSchemaFactory(saas_config).get_saas_schema() + schema.parse_obj({"account_type": ["checking", "savings"]}) + + def test_value_not_in_options(self, saas_config: SaaSConfig): + saas_config.connector_params = [ + ConnectorParam(name="account_type", options=["checking", "savings"]) + ] + schema = SaaSSchemaFactory(saas_config).get_saas_schema() + with pytest.raises(ValidationError) as exc: + schema.parse_obj({"account_type": "investment"}) + assert "'account_type' must be one of [checking, savings]" in str(exc.value) + + def test_value_not_in_options_with_multiselect(self, saas_config: SaaSConfig): + saas_config.connector_params = [ + ConnectorParam( + name="account_type", options=["checking", "savings"], multiselect=True + ) + ] + schema = SaaSSchemaFactory(saas_config).get_saas_schema() + with pytest.raises(ValidationError) as exc: + schema.parse_obj({"account_type": ["checking", "investment"]}) + assert ( + "[investment] are not valid options, 'account_type' must be a list of values from [checking, savings]" + in str(exc.value) + ) diff --git a/tests/ops/service/connectors/test_queryconfig.py b/tests/ops/service/connectors/test_queryconfig.py index 075db4e0d..5efdaccdd 100644 --- a/tests/ops/service/connectors/test_queryconfig.py +++ b/tests/ops/service/connectors/test_queryconfig.py @@ -1,9 +1,7 @@ -import json -from typing import Any, Dict, Optional, Set +from typing import Any, Dict, Set import pytest -from fidesops.ops.core.config import config from fidesops.ops.graph.config import ( CollectionAddress, FieldAddress, @@ -18,13 +16,10 @@ from fidesops.ops.schemas.dataset import FidesopsDataset from fidesops.ops.schemas.masking.masking_configuration import HashMaskingConfiguration from fidesops.ops.schemas.masking.masking_secrets import MaskingSecretCache, SecretType -from fidesops.ops.schemas.saas.saas_config import ParamValue, SaaSConfig, SaaSRequest -from fidesops.ops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams from fidesops.ops.service.connectors.query_config import ( MongoQueryConfig, SQLQueryConfig, ) -from fidesops.ops.service.connectors.saas_query_config import SaaSQueryConfig from fidesops.ops.service.masking.strategy.masking_strategy_hash import ( HashMaskingStrategy, ) @@ -611,373 +606,3 @@ def test_generate_update_stmt_multiple_rules( ["1988-01-10"], request_id=privacy_request.id )[0] ) - - -@pytest.mark.unit_saas -class TestSaaSQueryConfig: - @pytest.fixture(scope="function") - def combined_traversal( - self, saas_example_connection_config, saas_example_dataset_config - ): - merged_graph = saas_example_dataset_config.get_graph() - graph = DatasetGraph(merged_graph) - return Traversal(graph, {"email": "customer-1@example.com"}) - - def test_generate_query( - self, policy, combined_traversal, saas_example_connection_config - ): - saas_config = saas_example_connection_config.get_saas_config() - endpoints = saas_config.top_level_endpoint_dict - - member = combined_traversal.traversal_node_dict[ - CollectionAddress(saas_config.fides_key, "member") - ] - conversations = combined_traversal.traversal_node_dict[ - CollectionAddress(saas_config.fides_key, "conversations") - ] - messages = combined_traversal.traversal_node_dict[ - CollectionAddress(saas_config.fides_key, "messages") - ] - payment_methods = combined_traversal.traversal_node_dict[ - CollectionAddress(saas_config.fides_key, "payment_methods") - ] - - data_management = combined_traversal.traversal_node_dict[ - CollectionAddress(saas_config.fides_key, "data_management") - ] - - # static path with single query param - config = SaaSQueryConfig(member, endpoints, {}) - prepared_request: SaaSRequestParams = config.generate_query( - {"email": ["customer-1@example.com"]}, policy - ) - assert prepared_request.method == HTTPMethod.GET.value - assert prepared_request.path == "/3.0/search-members" - assert prepared_request.query_params == {"query": "customer-1@example.com"} - assert prepared_request.body is None - - # static path with multiple query params with default values - config = SaaSQueryConfig(conversations, endpoints, {}) - prepared_request = config.generate_query( - {"placeholder": ["customer-1@example.com"]}, policy - ) - assert prepared_request.method == HTTPMethod.GET.value - assert prepared_request.path == "/3.0/conversations" - assert prepared_request.query_params == {"count": 1000, "offset": 0} - assert prepared_request.body is None - - # dynamic path with no query params - config = SaaSQueryConfig(messages, endpoints, {}) - prepared_request = config.generate_query({"conversation_id": ["abc"]}, policy) - assert prepared_request.method == HTTPMethod.GET.value - assert prepared_request.path == "/3.0/conversations/abc/messages" - assert prepared_request.query_params == {} - assert prepared_request.body is None - - # header, query, and path params with connector param references - config = SaaSQueryConfig( - payment_methods, - endpoints, - {"api_version": "2.0", "page_size": 10, "api_key": "letmein"}, - ) - prepared_request = config.generate_query( - {"email": ["customer-1@example.com"]}, policy - ) - assert prepared_request.method == HTTPMethod.GET.value - assert prepared_request.path == "/2.0/payment_methods" - assert prepared_request.headers == { - "Content-Type": "application/json", - "On-Behalf-Of": "customer-1@example.com", - "Token": "Custom letmein", - } - assert prepared_request.query_params == { - "limit": "10", - "query": "customer-1@example.com", - } - assert prepared_request.body is None - - # query and path params with connector param references - config = SaaSQueryConfig( - payment_methods, endpoints, {"api_version": "2.0", "page_size": 10} - ) - prepared_request = config.generate_query( - {"email": ["customer-1@example.com"]}, policy - ) - assert prepared_request.method == HTTPMethod.GET.value - assert prepared_request.path == "/2.0/payment_methods" - assert prepared_request.query_params == { - "limit": "10", - "query": "customer-1@example.com", - } - - # using privacy_request_id placeholder - config = SaaSQueryConfig(data_management, endpoints, {}, None, privacy_request) - prepared_request = config.generate_query( - {"email": ["customer-1@example.com"]}, policy - ) - assert prepared_request.method == HTTPMethod.GET.value - assert prepared_request.path == f"/v1/privacy_request/{privacy_request.id}" - - def test_generate_update_stmt( - self, - erasure_policy_string_rewrite, - combined_traversal, - saas_example_connection_config, - ): - saas_config = saas_example_connection_config.get_saas_config() - endpoints = saas_config.top_level_endpoint_dict - update_request = endpoints["member"].requests.get("update") - - member = combined_traversal.traversal_node_dict[ - CollectionAddress(saas_config.fides_key, "member") - ] - - data_management = combined_traversal.traversal_node_dict[ - CollectionAddress(saas_config.fides_key, "data_management") - ] - - config = SaaSQueryConfig(member, endpoints, {}, update_request) - row = { - "id": "123", - "merge_fields": {"FNAME": "First", "LNAME": "Last"}, - "list_id": "abc", - } - - # build request by taking a row, masking it, and adding it to - # the body of a PUT request - prepared_request: SaaSRequestParams = config.generate_update_stmt( - row, erasure_policy_string_rewrite, privacy_request - ) - assert prepared_request.method == HTTPMethod.PUT.value - assert prepared_request.path == "/3.0/lists/abc/members/123" - assert prepared_request.headers == {"Content-Type": "application/json"} - assert prepared_request.query_params == {} - assert json.loads(prepared_request.body) == { - "merge_fields": {"FNAME": "MASKED", "LNAME": "MASKED"} - } - - # using privacy_request_id placeholder - config = SaaSQueryConfig(data_management, endpoints, {}, None, privacy_request) - prepared_request = config.generate_update_stmt( - row, erasure_policy_string_rewrite, privacy_request - ) - assert prepared_request.method == HTTPMethod.POST.value - assert prepared_request.path == "/v1/privacy_request/" - assert json.loads(prepared_request.body) == {"unique_id": privacy_request.id} - - def test_generate_update_stmt_custom_http_method( - self, - erasure_policy_string_rewrite, - combined_traversal, - saas_example_connection_config, - ): - saas_config: Optional[ - SaaSConfig - ] = saas_example_connection_config.get_saas_config() - saas_config.endpoints[2].requests.get("update").method = HTTPMethod.POST - endpoints = saas_config.top_level_endpoint_dict - - member = combined_traversal.traversal_node_dict[ - CollectionAddress(saas_config.fides_key, "member") - ] - update_request = endpoints["member"].requests.get("update") - - config = SaaSQueryConfig(member, endpoints, {}, update_request) - row = { - "id": "123", - "merge_fields": {"FNAME": "First", "LNAME": "Last"}, - "list_id": "abc", - } - - # build request by taking a row, masking it, and adding it to - # the body of a POST request - prepared_request: SaaSRequestParams = config.generate_update_stmt( - row, erasure_policy_string_rewrite, privacy_request - ) - assert prepared_request.method == HTTPMethod.POST.value - assert prepared_request.path == "/3.0/lists/abc/members/123" - assert prepared_request.headers == {"Content-Type": "application/json"} - assert prepared_request.query_params == {} - assert json.loads(prepared_request.body) == { - "merge_fields": {"FNAME": "MASKED", "LNAME": "MASKED"} - } - - def test_generate_update_stmt_with_request_body( - self, - erasure_policy_string_rewrite, - combined_traversal, - saas_example_connection_config, - ): - saas_config: Optional[ - SaaSConfig - ] = saas_example_connection_config.get_saas_config() - saas_config.endpoints[2].requests.get( - "update" - ).body = '{"properties": {, "list_id": ""}}' - body_param_value = ParamValue( - name="list_id", - type="body", - references=[ - { - "dataset": "saas_connector_example", - "field": "member.list_id", - "direction": "from", - } - ], - ) - saas_config.endpoints[2].requests.get("update").param_values.append( - body_param_value - ) - endpoints = saas_config.top_level_endpoint_dict - update_request = endpoints["member"].requests.get("update") - member = combined_traversal.traversal_node_dict[ - CollectionAddress(saas_config.fides_key, "member") - ] - payment_methods = combined_traversal.traversal_node_dict[ - CollectionAddress(saas_config.fides_key, "payment_methods") - ] - - config = SaaSQueryConfig(member, endpoints, {}, update_request) - row = { - "id": "123", - "merge_fields": {"FNAME": "First", "LNAME": "Last"}, - "list_id": "abc", - } - # build request by taking a row, masking it, and adding it to - # the body of a PUT request - prepared_request = config.generate_update_stmt( - row, erasure_policy_string_rewrite, privacy_request - ) - assert prepared_request == SaaSRequestParams( - method=HTTPMethod.PUT, - path="/3.0/lists/abc/members/123", - headers={"Content-Type": "application/json"}, - query_params={}, - body=json.dumps( - { - "properties": { - "merge_fields": {"FNAME": "MASKED", "LNAME": "MASKED"}, - "list_id": "abc", - } - } - ), - ) - - # update with connector_param reference - update_request = endpoints["payment_methods"].requests.get("update") - config = SaaSQueryConfig( - payment_methods, endpoints, {"api_version": "2.0"}, update_request - ) - row = {"type": "card", "customer_name": "First Last"} - prepared_request = config.generate_update_stmt( - row, erasure_policy_string_rewrite, privacy_request - ) - assert prepared_request.method == HTTPMethod.PUT.value - assert prepared_request.path == "/2.0/payment_methods" - assert prepared_request.headers == {"Content-Type": "application/json"} - assert prepared_request.query_params == {} - assert json.loads(prepared_request.body) == {"customer_name": "MASKED"} - - def test_generate_update_stmt_with_url_encoded_body( - self, - erasure_policy_string_rewrite, - combined_traversal, - saas_example_connection_config, - ): - saas_config: Optional[ - SaaSConfig - ] = saas_example_connection_config.get_saas_config() - endpoints = saas_config.top_level_endpoint_dict - customer = combined_traversal.traversal_node_dict[ - CollectionAddress(saas_config.fides_key, "customer") - ] - - # update with multidimensional urlcoding - # omit read-only fields and fields not defined in the dataset - # 'created' and 'id' are flagged as read-only and 'livemode' is not in the dataset - update_request = endpoints["customer"].requests.get("update") - config = SaaSQueryConfig(customer, endpoints, {}, update_request) - row = { - "id": 1, - "name": {"first": "A", "last": "B"}, - "created": 1649198338, - "livemode": False, - } - prepared_request = config.generate_update_stmt( - row, erasure_policy_string_rewrite, privacy_request - ) - assert prepared_request.method == HTTPMethod.POST.value - assert prepared_request.path == "/v1/customers/1" - assert prepared_request.headers == { - "Content-Type": "application/x-www-form-urlencoded" - } - assert prepared_request.query_params == {} - assert prepared_request.body == "name%5Bfirst%5D=MASKED&name%5Blast%5D=MASKED" - - def test_get_masking_request( - self, combined_traversal, saas_example_connection_config - ): - saas_config: Optional[ - SaaSConfig - ] = saas_example_connection_config.get_saas_config() - endpoints = saas_config.top_level_endpoint_dict - - member = combined_traversal.traversal_node_dict[ - CollectionAddress(saas_config.fides_key, "member") - ] - conversations = combined_traversal.traversal_node_dict[ - CollectionAddress(saas_config.fides_key, "conversations") - ] - messages = combined_traversal.traversal_node_dict[ - CollectionAddress(saas_config.fides_key, "messages") - ] - - query_config = SaaSQueryConfig(member, endpoints, {}) - saas_request = query_config.get_masking_request() - - # Assert we pulled the update method off of the member collection - assert saas_request.method == "PUT" - assert saas_request.path == "/3.0/lists//members/" - - # No update methods defined on other collections - query_config = SaaSQueryConfig(conversations, endpoints, {}) - saas_request = query_config.get_masking_request() - assert saas_request is None - - query_config = SaaSQueryConfig(messages, endpoints, {}) - saas_request = query_config.get_masking_request() - assert saas_request is None - - # Define delete request on conversations endpoint - endpoints["conversations"].requests["delete"] = SaaSRequest( - method="DELETE", path="/api/0///" - ) - # Delete endpoint not used because MASKING_STRICT is True - assert config.execution.masking_strict is True - - query_config = SaaSQueryConfig(conversations, endpoints, {}) - saas_request = query_config.get_masking_request() - assert saas_request is None - - # Override MASKING_STRICT to False - config.execution.masking_strict = False - - # Now delete endpoint is selected as conversations masking request - saas_request: SaaSRequest = query_config.get_masking_request() - assert saas_request.path == "/api/0///" - assert saas_request.method == "DELETE" - - # Define GDPR Delete - data_protection_request = SaaSRequest(method="PUT", path="/api/0/gdpr_delete") - query_config = SaaSQueryConfig( - conversations, endpoints, {}, data_protection_request - ) - - # Assert GDPR Delete takes priority over Delete - saas_request: SaaSRequest = query_config.get_masking_request() - assert saas_request.path == "/api/0/gdpr_delete" - assert saas_request.method == "PUT" - - # Reset - config.execution.masking_strict = True - del endpoints["conversations"].requests["delete"] diff --git a/tests/ops/service/connectors/test_saas_queryconfig.py b/tests/ops/service/connectors/test_saas_queryconfig.py new file mode 100644 index 000000000..d66fce394 --- /dev/null +++ b/tests/ops/service/connectors/test_saas_queryconfig.py @@ -0,0 +1,521 @@ +import json +from typing import List, Optional + +import pytest + +from fidesops.ops.core.config import config +from fidesops.ops.graph.config import CollectionAddress +from fidesops.ops.graph.graph import DatasetGraph +from fidesops.ops.graph.traversal import Traversal +from fidesops.ops.models.privacy_request import PrivacyRequest +from fidesops.ops.schemas.saas.saas_config import ParamValue, SaaSConfig, SaaSRequest +from fidesops.ops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams +from fidesops.ops.service.connectors.saas_query_config import SaaSQueryConfig + +privacy_request = PrivacyRequest(id="234544") + + +@pytest.mark.unit_saas +class TestSaaSQueryConfig: + @pytest.fixture(scope="function") + def combined_traversal( + self, saas_example_connection_config, saas_example_dataset_config + ): + merged_graph = saas_example_dataset_config.get_graph() + graph = DatasetGraph(merged_graph) + return Traversal(graph, {"email": "customer-1@example.com"}) + + def test_generate_requests( + self, policy, combined_traversal, saas_example_connection_config + ): + saas_config = saas_example_connection_config.get_saas_config() + endpoints = saas_config.top_level_endpoint_dict + + member = combined_traversal.traversal_node_dict[ + CollectionAddress(saas_config.fides_key, "member") + ] + conversations = combined_traversal.traversal_node_dict[ + CollectionAddress(saas_config.fides_key, "conversations") + ] + messages = combined_traversal.traversal_node_dict[ + CollectionAddress(saas_config.fides_key, "messages") + ] + payment_methods = combined_traversal.traversal_node_dict[ + CollectionAddress(saas_config.fides_key, "payment_methods") + ] + + # static path with single query param + config = SaaSQueryConfig(member, endpoints, {}) + prepared_request: SaaSRequestParams = config.generate_requests( + {"fidesops_grouped_inputs": [], "email": ["customer-1@example.com"]}, policy + )[0] + assert prepared_request.method == HTTPMethod.GET.value + assert prepared_request.path == "/3.0/search-members" + assert prepared_request.query_params == {"query": "customer-1@example.com"} + assert prepared_request.body is None + + # static path with multiple query params with default values + config = SaaSQueryConfig(conversations, endpoints, {}) + prepared_request = config.generate_requests( + { + "fidesops_grouped_inputs": [], + "placeholder": ["adaptors.india@ethyca.com"], + }, + policy, + )[0] + assert prepared_request.method == HTTPMethod.GET.value + assert prepared_request.path == "/3.0/conversations" + assert prepared_request.query_params == {"count": 1000, "offset": 0} + assert prepared_request.body is None + + # dynamic path with no query params + config = SaaSQueryConfig(messages, endpoints, {}) + prepared_request = config.generate_requests( + {"fidesops_grouped_inputs": [], "conversation_id": ["abc"]}, policy + )[0] + assert prepared_request.method == HTTPMethod.GET.value + assert prepared_request.path == "/3.0/conversations/abc/messages" + assert prepared_request.query_params == {} + assert prepared_request.body is None + + # header, query, and path params with connector param references + config = SaaSQueryConfig( + payment_methods, + endpoints, + {"api_version": "2.0", "page_size": 10, "api_key": "letmein"}, + ) + prepared_request = config.generate_requests( + {"fidesops_grouped_inputs": [], "email": ["customer-1@example.com"]}, policy + )[0] + assert prepared_request.method == HTTPMethod.GET.value + assert prepared_request.path == "/2.0/payment_methods" + assert prepared_request.headers == { + "Content-Type": "application/json", + "On-Behalf-Of": "customer-1@example.com", + "Token": "Custom letmein", + } + assert prepared_request.query_params == { + "limit": "10", + "query": "customer-1@example.com", + } + assert prepared_request.body is None + + # query and path params with connector param references + config = SaaSQueryConfig( + payment_methods, + endpoints, + {"api_version": "2.0", "page_size": 10, "api_key": "letmein"}, + ) + prepared_request: SaaSRequestParams = config.generate_requests( + {"fidesops_grouped_inputs": [], "email": ["customer-1@example.com"]}, policy + )[0] + assert prepared_request.method == HTTPMethod.GET.value + assert prepared_request.path == "/2.0/payment_methods" + assert prepared_request.query_params == { + "limit": "10", + "query": "customer-1@example.com", + } + + def test_generate_update_stmt( + self, + erasure_policy_string_rewrite, + combined_traversal, + saas_example_connection_config, + ): + saas_config = saas_example_connection_config.get_saas_config() + endpoints = saas_config.top_level_endpoint_dict + update_request = endpoints["member"].requests.get("update") + + member = combined_traversal.traversal_node_dict[ + CollectionAddress(saas_config.fides_key, "member") + ] + + config = SaaSQueryConfig(member, endpoints, {}, update_request) + row = { + "id": "123", + "merge_fields": {"FNAME": "First", "LNAME": "Last"}, + "list_id": "abc", + } + + # build request by taking a row, masking it, and adding it to + # the body of a PUT request + prepared_request: SaaSRequestParams = config.generate_update_stmt( + row, erasure_policy_string_rewrite, privacy_request + ) + assert prepared_request.method == HTTPMethod.PUT.value + assert prepared_request.path == "/3.0/lists/abc/members/123" + assert prepared_request.headers == {"Content-Type": "application/json"} + assert prepared_request.query_params == {} + assert ( + prepared_request.body + == '{\n "merge_fields": {"FNAME": "MASKED", "LNAME": "MASKED"}\n}\n' + ) + + def test_generate_update_stmt_custom_http_method( + self, + erasure_policy_string_rewrite, + combined_traversal, + saas_example_connection_config, + ): + saas_config: Optional[ + SaaSConfig + ] = saas_example_connection_config.get_saas_config() + saas_config.endpoints[2].requests.get("update").method = HTTPMethod.POST + endpoints = saas_config.top_level_endpoint_dict + + member = combined_traversal.traversal_node_dict[ + CollectionAddress(saas_config.fides_key, "member") + ] + update_request = endpoints["member"].requests.get("update") + + config = SaaSQueryConfig(member, endpoints, {}, update_request) + row = { + "id": "123", + "merge_fields": {"FNAME": "First", "LNAME": "Last"}, + "list_id": "abc", + } + + # build request by taking a row, masking it, and adding it to + # the body of a POST request + prepared_request: SaaSRequestParams = config.generate_update_stmt( + row, erasure_policy_string_rewrite, privacy_request + ) + assert prepared_request.method == HTTPMethod.POST.value + assert prepared_request.path == "/3.0/lists/abc/members/123" + assert prepared_request.headers == {"Content-Type": "application/json"} + assert prepared_request.query_params == {} + assert ( + prepared_request.body + == '{\n "merge_fields": {"FNAME": "MASKED", "LNAME": "MASKED"}\n}\n' + ) + + def test_generate_update_stmt_with_request_body( + self, + erasure_policy_string_rewrite, + combined_traversal, + saas_example_connection_config, + ): + saas_config: Optional[ + SaaSConfig + ] = saas_example_connection_config.get_saas_config() + saas_config.endpoints[2].requests.get( + "update" + ).body = '{"properties": {, "list_id": ""}}' + body_param_value = ParamValue( + name="list_id", + type="body", + references=[ + { + "dataset": "saas_connector_example", + "field": "member.list_id", + "direction": "from", + } + ], + ) + saas_config.endpoints[2].requests.get("update").param_values.append( + body_param_value + ) + endpoints = saas_config.top_level_endpoint_dict + update_request = endpoints["member"].requests.get("update") + member = combined_traversal.traversal_node_dict[ + CollectionAddress(saas_config.fides_key, "member") + ] + payment_methods = combined_traversal.traversal_node_dict[ + CollectionAddress(saas_config.fides_key, "payment_methods") + ] + + config = SaaSQueryConfig(member, endpoints, {}, update_request) + row = { + "id": "123", + "merge_fields": {"FNAME": "First", "LNAME": "Last"}, + "list_id": "abc", + } + # build request by taking a row, masking it, and adding it to + # the body of a PUT request + prepared_request = config.generate_update_stmt( + row, erasure_policy_string_rewrite, privacy_request + ) + assert prepared_request == SaaSRequestParams( + method=HTTPMethod.PUT, + path="/3.0/lists/abc/members/123", + headers={"Content-Type": "application/json"}, + query_params={}, + body=json.dumps( + { + "properties": { + "merge_fields": {"FNAME": "MASKED", "LNAME": "MASKED"}, + "list_id": "abc", + } + } + ), + ) + + # update with connector_param reference + update_request = endpoints["payment_methods"].requests.get("update") + config = SaaSQueryConfig( + payment_methods, endpoints, {"api_version": "2.0"}, update_request + ) + row = {"type": "card", "customer_name": "First Last"} + prepared_request = config.generate_update_stmt( + row, erasure_policy_string_rewrite, privacy_request + ) + assert prepared_request.method == HTTPMethod.PUT.value + assert prepared_request.path == "/2.0/payment_methods" + assert prepared_request.headers == {"Content-Type": "application/json"} + assert prepared_request.query_params == {} + assert prepared_request.body == '{\n "customer_name": "MASKED"\n}\n' + + def test_generate_update_stmt_with_url_encoded_body( + self, + erasure_policy_string_rewrite, + combined_traversal, + saas_example_connection_config, + ): + saas_config: Optional[ + SaaSConfig + ] = saas_example_connection_config.get_saas_config() + endpoints = saas_config.top_level_endpoint_dict + customer = combined_traversal.traversal_node_dict[ + CollectionAddress(saas_config.fides_key, "customer") + ] + + # update with multidimensional urlcoding + # omit read-only fields and fields not defined in the dataset + # 'created' and 'id' are flagged as read-only and 'livemode' is not in the dataset + update_request = endpoints["customer"].requests.get("update") + config = SaaSQueryConfig(customer, endpoints, {}, update_request) + row = { + "id": 1, + "name": {"first": "A", "last": "B"}, + "created": 1649198338, + "livemode": False, + } + prepared_request = config.generate_update_stmt( + row, erasure_policy_string_rewrite, privacy_request + ) + assert prepared_request.method == HTTPMethod.POST.value + assert prepared_request.path == "/v1/customers/1" + assert prepared_request.headers == { + "Content-Type": "application/x-www-form-urlencoded" + } + assert prepared_request.query_params == {} + assert prepared_request.body == "name%5Bfirst%5D=MASKED&name%5Blast%5D=MASKED" + + def test_get_masking_request( + self, combined_traversal, saas_example_connection_config + ): + saas_config: Optional[ + SaaSConfig + ] = saas_example_connection_config.get_saas_config() + endpoints = saas_config.top_level_endpoint_dict + + member = combined_traversal.traversal_node_dict[ + CollectionAddress(saas_config.fides_key, "member") + ] + conversations = combined_traversal.traversal_node_dict[ + CollectionAddress(saas_config.fides_key, "conversations") + ] + messages = combined_traversal.traversal_node_dict[ + CollectionAddress(saas_config.fides_key, "messages") + ] + + query_config = SaaSQueryConfig(member, endpoints, {}) + saas_request = query_config.get_masking_request() + + # Assert we pulled the update method off of the member collection + assert saas_request.method == "PUT" + assert saas_request.path == "/3.0/lists//members/" + + # No update methods defined on other collections + query_config = SaaSQueryConfig(conversations, endpoints, {}) + saas_request = query_config.get_masking_request() + assert saas_request is None + + query_config = SaaSQueryConfig(messages, endpoints, {}) + saas_request = query_config.get_masking_request() + assert saas_request is None + + # Define delete request on conversations endpoint + endpoints["conversations"].requests["delete"] = SaaSRequest( + method="DELETE", path="/api/0///" + ) + # Delete endpoint not used because masking_strict is True + assert config.execution.masking_strict is True + + query_config = SaaSQueryConfig(conversations, endpoints, {}) + saas_request = query_config.get_masking_request() + assert saas_request is None + + # Override masking_strict to False + config.execution.masking_strict = False + + # Now delete endpoint is selected as conversations masking request + saas_request: SaaSRequest = query_config.get_masking_request() + assert saas_request.path == "/api/0///" + assert saas_request.method == "DELETE" + + # Define GDPR Delete + data_protection_request = SaaSRequest(method="PUT", path="/api/0/gdpr_delete") + query_config = SaaSQueryConfig( + conversations, endpoints, {}, data_protection_request + ) + + # Assert GDPR Delete takes priority over Delete + saas_request: SaaSRequest = query_config.get_masking_request() + assert saas_request.path == "/api/0/gdpr_delete" + assert saas_request.method == "PUT" + + # Reset + config.execution.masking_strict = True + del endpoints["conversations"].requests["delete"] + + def test_list_param_values( + self, combined_traversal, saas_example_connection_config, policy + ): + saas_config: Optional[ + SaaSConfig + ] = saas_example_connection_config.get_saas_config() + endpoints = saas_config.top_level_endpoint_dict + + accounts = combined_traversal.traversal_node_dict[ + CollectionAddress(saas_config.fides_key, "accounts") + ] + + config = SaaSQueryConfig( + accounts, + endpoints, + { + "api_version": "2.0", + "page_size": 10, + "api_key": "letmein", + "account_types": ["checking", "savings", "investment"], + }, + ) + + prepared_requests: List[SaaSRequestParams] = config.generate_requests( + { + "email": ["customer-1@example.com"], + "fidesops_grouped_inputs": [ + {"customer_id": ["1"], "customer_name": ["a"]} + ], + }, + policy, + ) + assert len(prepared_requests) == 3 + + config = SaaSQueryConfig( + accounts, + endpoints, + { + "api_version": "2.0", + "page_size": 10, + "api_key": "letmein", + "account_types": ["checking"], + }, + ) + prepared_requests: List[SaaSRequestParams] = config.generate_requests( + { + "email": ["customer-1@example.com"], + "fidesops_grouped_inputs": [ + {"customer_id": ["1"], "customer_name": ["a"]} + ], + }, + policy, + ) + assert len(prepared_requests) == 1 + + def test_list_inputs( + self, combined_traversal, saas_example_connection_config, policy + ): + """ + This demonstrates that multivalue connector params wont't generate + more prepared_requests if they are not used by the request + """ + + saas_config: Optional[ + SaaSConfig + ] = saas_example_connection_config.get_saas_config() + endpoints = saas_config.top_level_endpoint_dict + + mailing_lists = combined_traversal.traversal_node_dict[ + CollectionAddress(saas_config.fides_key, "mailing_lists") + ] + + config = SaaSQueryConfig( + mailing_lists, + endpoints, + { + "api_version": "2.0", + "page_size": 10, + "api_key": "letmein", + "account_types": ["checking", "savings", "investment"], + }, + ) + + prepared_requests: List[SaaSRequestParams] = config.generate_requests( + { + "email": ["customer-1@example.com"], + "list_id": [[1, 2, 3]], + }, + policy, + ) + assert len(prepared_requests) == 3 + + +class TestGenerateProductList: + def test_vector_values(self): + assert SaaSQueryConfig._generate_product_list( + {"first": ["a", "b", "c"], "second": [1, 2, 3]} + == [ + {"first": "a", "second": 1}, + {"first": "a", "second": 2}, + {"first": "a", "second": 3}, + {"first": "b", "second": 1}, + {"first": "b", "second": 2}, + {"first": "b", "second": 3}, + {"first": "c", "second": 1}, + {"first": "c", "second": 2}, + {"first": "c", "second": 3}, + ] + ) + + def test_with_scalar_values(self): + assert SaaSQueryConfig._generate_product_list( + {"first": "a", "second": 1} == [{"first": "a", "second": 1}] + ) + + def test_with_empty_list_with_scalar_value(self): + assert SaaSQueryConfig._generate_product_list( + {"first": [], "second": 1} == [{"second": 1}] + ) + + def test_with_empty_list_with_vector_value(self): + assert SaaSQueryConfig._generate_product_list( + {"first": [], "second": [1, 2, 3]} + == [{"second": 1}, {"second": 2}, {"second": 3}] + ) + + def test_scalar_with_vector_values(self): + assert SaaSQueryConfig._generate_product_list( + {"first": "a", "second": [1, 2, 3]} + == [ + {"first": "a", "second": 1}, + {"first": "a", "second": 2}, + {"first": "a", "second": 3}, + ] + ) + + def test_multiple_dicts_with_vector_values(self): + assert SaaSQueryConfig._generate_product_list( + {"first": ["a", "b", "c"]}, {"second": [1, 2, 3]} + ) == [ + {"first": "a", "second": 1}, + {"first": "a", "second": 2}, + {"first": "a", "second": 3}, + {"first": "b", "second": 1}, + {"first": "b", "second": 2}, + {"first": "b", "second": 3}, + {"first": "c", "second": 1}, + {"first": "c", "second": 2}, + {"first": "c", "second": 3}, + ]