From 17051ecde2680be16a2e934b6a66b3994eb929dd Mon Sep 17 00:00:00 2001 From: Leonid Maksimov Date: Thu, 21 Sep 2023 19:58:47 +0200 Subject: [PATCH] LITE-28666 search by parameter value was fixed ignore FF requests without changes in case of duplications ignore FF requests that was approved after batch.context.period.end --- .../lookup_ff_request/mixins.py | 54 ++- .../lookup_ff_request/utils.py | 25 ++ .../lookup_subscription/mixins.py | 4 +- ...lookup_ff_request.2d9834915db04892e9cf.js} | 17 +- .../transformations/lookup_ff_request.html | 2 +- .../test_lookup_ff_requests.py | 329 ++++++++++++++++++ .../transformations/lookup_ff_request.js | 17 +- 7 files changed, 418 insertions(+), 30 deletions(-) rename connect_transformations/static/transformations/{lookup_ff_request.5ad2ce11ef60a735c477.js => lookup_ff_request.2d9834915db04892e9cf.js} (98%) create mode 100644 tests/transformations/test_lookup_ff_requests.py diff --git a/connect_transformations/lookup_ff_request/mixins.py b/connect_transformations/lookup_ff_request/mixins.py index 624cbb6..e71f5e4 100644 --- a/connect_transformations/lookup_ff_request/mixins.py +++ b/connect_transformations/lookup_ff_request/mixins.py @@ -5,16 +5,21 @@ # from typing import Dict -from connect.client import AsyncConnectClient, ClientError +from connect.client import AsyncConnectClient from connect.eaas.core.decorators import router, transformation from connect.eaas.core.inject.asynchronous import get_installation_client from connect.eaas.core.responses import RowTransformationResponse from fastapi import Depends -from connect_transformations.constants import MAX_API_CALL_CONNECTION_ERROR_RETRIES, SEPARATOR +from connect_transformations.constants import SEPARATOR from connect_transformations.lookup_ff_request.exceptions import FFRequestLookupError from connect_transformations.lookup_ff_request.models import Configuration, SubscriptionParameter -from connect_transformations.lookup_ff_request.utils import validate_lookup_ff_request +from connect_transformations.lookup_ff_request.utils import ( + FF_REQ_COMMON_FILTERS, + FF_REQ_SELECT, + filter_requests_with_changes, + validate_lookup_ff_request, +) from connect_transformations.models import Error, ValidationResult from connect_transformations.utils import deep_itemgetter, is_input_column_nullable @@ -47,8 +52,8 @@ async def lookup_ff_request( if self.settings.get('asset_type'): lookup[f'asset.{self.settings["asset_type"]}'] = row[self.settings['asset_column']] - lookup['params.name'] = self.settings['parameter']['name'] - lookup['params.value'] = row[self.settings['parameter_column']] + lookup['asset.params.name'] = self.settings['parameter']['name'] + lookup['asset.params.value'] = row[self.settings['parameter_column']] try: request = await self.get_request(lookup) @@ -71,8 +76,9 @@ def extract_row_from_request(self, request, output_columns, item_id): value = None attr = col_config['attribute'] if attr == 'asset.parameter.value': + param_name = col_config['parameter_name'] for param_data in request['asset']['params']: - if param_data['name'] == self.settings['parameter']['name']: + if param_data['name'] == param_name: value = param_data['value'] break elif attr.startswith('asset.items.'): @@ -96,7 +102,12 @@ def extract_item_attrs_from_request(self, item_attrs, row, request, item_id_valu continue for col_name, item_attr in item_attrs: - item_value = str(item.get(item_attr, '')) + if item_attr == 'quantity_delta': + item_value = int(item['quantity']) - int(item['old_quantity']) + else: + item_value = item.get(item_attr, '') + + item_value = str(item_value) row[col_name] += f'{SEPARATOR}{item_value}' if row[col_name] else item_value async def get_request(self, lookup): @@ -108,24 +119,33 @@ async def get_request(self, lookup): except KeyError: pass - for attempts_left in range(MAX_API_CALL_CONNECTION_ERROR_RETRIES, -1, -1): - try: - result = await self.retrieve_ff_requests(lookup) - break - except ClientError: - if not attempts_left: - raise - continue + result = await self.retrieve_ff_requests(lookup) await self.acache_put(k, result) return result async def retrieve_ff_requests(self, lookup): - requests = self.installation_client.requests.filter(**lookup).order_by('-created') + batch_context = self.transformation_request['batch']['context'] + period_end = batch_context.get('period', {}).get('end') + additional_filters = {} + if period_end: + additional_filters['updated__lt'] = period_end + + requests = self.installation_client.requests.filter( + **FF_REQ_COMMON_FILTERS, + **lookup, + **additional_filters, + ).select( + *FF_REQ_SELECT, + ).order_by('-updated') + requests = [r async for r in requests] + + if len(requests) > 1: + requests = filter_requests_with_changes(requests) result = None - async for item in requests: + for item in requests: if result is None: result = item elif self.settings.get('action_if_multiple') == 'leave_empty': diff --git a/connect_transformations/lookup_ff_request/utils.py b/connect_transformations/lookup_ff_request/utils.py index 870d679..da53422 100644 --- a/connect_transformations/lookup_ff_request/utils.py +++ b/connect_transformations/lookup_ff_request/utils.py @@ -19,6 +19,11 @@ NOT_FOUND_CHOICES = ['leave_empty', 'fail'] MULTIPLE_CHOICES = ['leave_empty', 'fail', 'use_most_actual'] +FF_REQ_COMMON_FILTERS = { + 'status': 'approved', +} +FF_REQ_SELECT = ['-activation_key', '-template'] + def validate_lookup_ff_request(data): # noqa: CCR001 data = data.dict(by_alias=True) @@ -119,3 +124,23 @@ def validate_lookup_ff_request(data): # noqa: CCR001 return { 'overview': overview, } + + +def filter_requests_with_changes(requests): + """ FF requests without any changes can be produced during synchronization. + Exclude them. + """ + filtered_requests = [] + for request in requests: + changed = False + for item_data in request['asset']['items']: + if item_data['quantity'] != item_data['old_quantity']: + changed = True + break + if changed: + filtered_requests.append(request) + + if not filtered_requests: + return requests[0] + + return filtered_requests diff --git a/connect_transformations/lookup_subscription/mixins.py b/connect_transformations/lookup_subscription/mixins.py index d2d4c9d..8353ca2 100644 --- a/connect_transformations/lookup_subscription/mixins.py +++ b/connect_transformations/lookup_subscription/mixins.py @@ -107,12 +107,12 @@ def extract_row_from_subscription(self, subscription, output_columns): row[col_name] = value if item_attrs: - self.extract_item_attrs_from_sutbscription(item_attrs, row, subscription) + self.extract_item_attrs_from_subscription(item_attrs, row, subscription) return row @staticmethod - def extract_item_attrs_from_sutbscription(item_attrs, row, subscription): + def extract_item_attrs_from_subscription(item_attrs, row, subscription): for item in subscription['items']: if ( item.get('item_type') == 'reservation' diff --git a/connect_transformations/static/transformations/lookup_ff_request.5ad2ce11ef60a735c477.js b/connect_transformations/static/transformations/lookup_ff_request.2d9834915db04892e9cf.js similarity index 98% rename from connect_transformations/static/transformations/lookup_ff_request.5ad2ce11ef60a735c477.js rename to connect_transformations/static/transformations/lookup_ff_request.2d9834915db04892e9cf.js index cde8abe..c5bd544 100644 --- a/connect_transformations/static/transformations/lookup_ff_request.5ad2ce11ef60a735c477.js +++ b/connect_transformations/static/transformations/lookup_ff_request.2d9834915db04892e9cf.js @@ -481,7 +481,15 @@ const AVAILABLE_FF_REQUEST_ATTRS = [ }, { value: 'asset.items.quantity', - label: 'Product item quantities', + label: 'Product item quantity', + }, + { + value: 'asset.items.old_quantity', + label: 'Product item previous quantity', + }, + { + value: 'asset.items.quantity_delta', + label: 'Product item quantity delta', }, ]; @@ -531,7 +539,7 @@ const createOutputRow = (parent, index, column, parameters, columnConfigs) => { }); const handleSourceSelectChange = () => { - if (sourceSelect.value === 'parameter.value') { + if (sourceSelect.value === 'asset.parameter.value') { paramNameSelect.style.display = 'block'; paramNameSelect.style.maxWidth = 'calc(28% - 10px)'; sourceSelect.style.maxWidth = 'calc(25% - 10px)'; @@ -573,7 +581,6 @@ const lookupFFRequest = (app) => { global_id: 'Item Global ID', mpn: 'Item MPN', all: 'Include all', - skip: 'Skip items', }; hideComponent('loader'); @@ -671,7 +678,7 @@ const lookupFFRequest = (app) => { }); document.getElementById('item').addEventListener('change', () => { - if (['skip', 'all'].includes(document.getElementById('item').value)) { + if (document.getElementById('item').value === 'all') { document.getElementById('item_column_group').style.display = 'none'; } else { document.getElementById('item_column_group').style.display = 'block'; @@ -728,7 +735,7 @@ const lookupFFRequest = (app) => { attribute: colSource, }; - if (colSource === 'parameter.value') { + if (colSource === 'asset.parameter.value') { const paramName = document.getElementById(`source-param-select-${index}`).value; columnConfig.parameter_name = paramName; } diff --git a/connect_transformations/static/transformations/lookup_ff_request.html b/connect_transformations/static/transformations/lookup_ff_request.html index a87ffa8..a6058a9 100644 --- a/connect_transformations/static/transformations/lookup_ff_request.html +++ b/connect_transformations/static/transformations/lookup_ff_request.html @@ -1 +1 @@ -Transformations/Lookup FF request
Supported search by parameter and optional search by Subscription ID or Subscription External ID. Result contains only one item information, matching selected criteria.

Choose search by Subscription ID

Choose search by parameter

Choose Item match criteria

Choose what to do if match is not found:

Choose what to do if multiple matches are found:

\ No newline at end of file +Transformations/Lookup FF request
Supported search by parameter and optional search by Subscription ID or Subscription External ID. Result contains only one item information, matching selected criteria.

Choose search by Subscription ID

Choose search by parameter

Choose Item match criteria

Choose what to do if match is not found:

Choose what to do if multiple matches are found:

\ No newline at end of file diff --git a/tests/transformations/test_lookup_ff_requests.py b/tests/transformations/test_lookup_ff_requests.py new file mode 100644 index 0000000..d37a438 --- /dev/null +++ b/tests/transformations/test_lookup_ff_requests.py @@ -0,0 +1,329 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) 2023, CloudBlue LLC +# All rights reserved. +# +import pytest +from connect.eaas.core.enums import ResultType + +from connect_transformations.lookup_ff_request.utils import FF_REQ_COMMON_FILTERS, FF_REQ_SELECT +from connect_transformations.transformations import StandardTransformationsApplication + + +@pytest.mark.asyncio +async def test_lookup_request_full( + mocker, + async_connect_client, + async_client_mocker_factory, +): + client = async_client_mocker_factory(base_url=async_connect_client.endpoint) + + client.requests.filter( + **FF_REQ_COMMON_FILTERS, + **{ + 'asset.id': 'AS-12311', + 'asset.params.name': 'param_name', + 'asset.params.value': 'ParamValue', + 'updated__lt': '2022-01-31T23:59:59', + }, + ).select( + *FF_REQ_SELECT, + ).order_by('-updated').mock(return_value=[ + { + 'id': 'PR-123-123', + 'status': 'approved', + 'asset': { + 'id': 'AS-12311', + 'params': [{ + 'name': 'param_name', + 'value': 'ParamValue', + }], + 'items': [ + {'id': 'i1', 'mpn': 'm1', 'quantity': 11, 'old_quantity': 0}, + {'id': 'i2', 'mpn': 'm2', 'quantity': 0, 'old_quantity': 0}, + {'id': 'i3', 'mpn': 'm3', 'quantity': 12, 'old_quantity': 5}, + ], + }, + }, + # FF request without changes + { + 'id': 'PR-123-234', + 'status': 'approved', + 'asset': { + 'params': [{ + 'name': 'param_name', + 'value': 'ParamValue', + }], + 'items': [ + {'id': 'i1', 'mpn': 'm1', 'quantity': 11, 'old_quantity': 11}, + {'id': 'i2', 'mpn': 'm2', 'quantity': 0, 'old_quantity': 0}, + {'id': 'i3', 'mpn': 'm3', 'quantity': 12, 'old_quantity': 12}, + ], + }, + }, + ]) + + m = mocker.MagicMock() + app = StandardTransformationsApplication(m, m, m) + app.installation_client = async_connect_client + app.transformation_request = { + 'batch': { + 'context': { + 'period': { + 'start': '2022-01-01T00:00:00', + 'end': '2022-01-31T23:59:59', + }, + }, + }, + 'transformation': { + 'settings': { + "item": { + "id": "mpn", + "name": "Item MPN", + }, + "parameter": { + "id": "PRM-000-048-001-0003", + "name": "param_name", + }, + "asset_type": "id", + "asset_column": "AssetID", + "item_column": "ItemMPN", + "output_config": { + "A": { + "attribute": "asset.parameter.value", + "parameter_name": "param_name", + }, + "Status": { + "attribute": "status", + }, + "Quantity": { + "attribute": "asset.items.quantity", + }, + "Old Quantity": { + "attribute": "asset.items.old_quantity", + }, + }, + "parameter_column": "ParamName", + "action_if_multiple": "fail", + "action_if_not_found": "fail", + }, + 'columns': { + 'input': [ + {'name': 'AssetID', 'nullable': False}, + {'name': 'ParamName', 'nullable': False}, + {'name': 'ItemMPN', 'nullable': False}, + ], + 'output': [ + {'name': 'A'}, + ], + }, + }, + } + response = await app.lookup_ff_request({ + 'AssetID': 'AS-12311', + 'ParamName': 'ParamValue', + 'ItemMPN': 'm3', + }) + assert response.status == ResultType.SUCCESS, response.output + assert response.transformed_row == { + 'A': 'ParamValue', + 'Status': 'approved', + 'Quantity': '12', + 'Old Quantity': '5', + } + + +@pytest.mark.asyncio +async def test_lookup_request_wo_asset( + mocker, + async_connect_client, + async_client_mocker_factory, +): + client = async_client_mocker_factory(base_url=async_connect_client.endpoint) + + client.requests.filter( + **FF_REQ_COMMON_FILTERS, + **{ + 'asset.params.name': 'param_name', + 'asset.params.value': 'ParamValue', + }, + ).select( + *FF_REQ_SELECT, + ).order_by('-updated').mock(return_value=[ + { + 'id': 'PR-123-123', + 'status': 'approved', + 'asset': { + 'id': 'AS-12311', + 'params': [{ + 'name': 'param_name', + 'value': 'ParamValue', + }], + 'items': [ + {'id': 'i1', 'mpn': 'm1', 'quantity': 11, 'old_quantity': 0}, + {'id': 'i2', 'mpn': 'm2', 'quantity': 0, 'old_quantity': 0}, + {'id': 'i3', 'mpn': 'm3', 'quantity': 12, 'old_quantity': 5}, + ], + }, + }, + ]) + + m = mocker.MagicMock() + app = StandardTransformationsApplication(m, m, m) + app.installation_client = async_connect_client + app.transformation_request = { + 'batch': { + 'context': {}, + }, + 'transformation': { + 'settings': { + "item": { + "id": "mpn", + "name": "Item MPN", + }, + "parameter": { + "id": "PRM-000-048-001-0003", + "name": "param_name", + }, + "asset_type": None, + "asset_column": None, + "item_column": "ItemMPN", + "output_config": { + "A": { + "attribute": "asset.parameter.value", + "parameter_name": "param_name", + }, + "Status": { + "attribute": "status", + }, + "Quantity": { + "attribute": "asset.items.quantity", + }, + "Old Quantity": { + "attribute": "asset.items.old_quantity", + }, + }, + "parameter_column": "ParamName", + "action_if_multiple": "fail", + "action_if_not_found": "fail", + }, + 'columns': { + 'input': [ + {'name': 'ParamName', 'nullable': False}, + {'name': 'ItemMPN', 'nullable': False}, + ], + 'output': [ + {'name': 'A'}, + ], + }, + }, + } + response = await app.lookup_ff_request({ + 'ParamName': 'ParamValue', + 'ItemMPN': 'm3', + }) + assert response.status == ResultType.SUCCESS, response.output + assert response.transformed_row == { + 'A': 'ParamValue', + 'Status': 'approved', + 'Quantity': '12', + 'Old Quantity': '5', + } + + +@pytest.mark.asyncio +async def test_lookup_request_all_items( + mocker, + async_connect_client, + async_client_mocker_factory, +): + client = async_client_mocker_factory(base_url=async_connect_client.endpoint) + + client.requests.filter( + **FF_REQ_COMMON_FILTERS, + **{ + 'asset.params.name': 'param_name', + 'asset.params.value': 'ParamValue', + }, + ).select( + *FF_REQ_SELECT, + ).order_by('-updated').mock(return_value=[ + { + 'id': 'PR-123-123', + 'status': 'approved', + 'asset': { + 'id': 'AS-12311', + 'params': [{ + 'name': 'param_name', + 'value': 'ParamValue', + }], + 'items': [ + {'id': 'i1', 'mpn': 'm1', 'quantity': 11, 'old_quantity': 0}, + {'id': 'i2', 'mpn': 'm2', 'quantity': 0, 'old_quantity': 0}, + {'id': 'i3', 'mpn': 'm3', 'quantity': 12, 'old_quantity': 5}, + ], + }, + }, + ]) + + m = mocker.MagicMock() + app = StandardTransformationsApplication(m, m, m) + app.installation_client = async_connect_client + app.transformation_request = { + 'batch': { + 'context': {}, + }, + 'transformation': { + 'settings': { + "item": { + "id": "all", + "name": "Include all", + }, + "parameter": { + "id": "PRM-000-048-001-0003", + "name": "param_name", + }, + "asset_type": None, + "asset_column": None, + "item_column": "ParamName", + "output_config": { + "A": { + "attribute": "asset.parameter.value", + "parameter_name": "param_name", + }, + "Status": { + "attribute": "status", + }, + "Quantity": { + "attribute": "asset.items.quantity", + }, + "Old Quantity": { + "attribute": "asset.items.old_quantity", + }, + }, + "parameter_column": "ParamName", + "action_if_multiple": "fail", + "action_if_not_found": "fail", + }, + 'columns': { + 'input': [ + {'name': 'ParamName', 'nullable': False}, + {'name': 'ItemMPN', 'nullable': False}, + ], + 'output': [ + {'name': 'A'}, + ], + }, + }, + } + response = await app.lookup_ff_request({ + 'ParamName': 'ParamValue', + 'ItemMPN': 'm3', + }) + assert response.status == ResultType.SUCCESS, response.output + assert response.transformed_row == { + 'A': 'ParamValue', + 'Status': 'approved', + 'Quantity': '11;0;12', + 'Old Quantity': '0;0;5', + } diff --git a/ui/src/pages/transformations/lookup_ff_request.js b/ui/src/pages/transformations/lookup_ff_request.js index 4450a8b..48075c8 100644 --- a/ui/src/pages/transformations/lookup_ff_request.js +++ b/ui/src/pages/transformations/lookup_ff_request.js @@ -174,7 +174,15 @@ const AVAILABLE_FF_REQUEST_ATTRS = [ }, { value: 'asset.items.quantity', - label: 'Product item quantities', + label: 'Product item quantity', + }, + { + value: 'asset.items.old_quantity', + label: 'Product item previous quantity', + }, + { + value: 'asset.items.quantity_delta', + label: 'Product item quantity delta', }, ]; @@ -224,7 +232,7 @@ const createOutputRow = (parent, index, column, parameters, columnConfigs) => { }); const handleSourceSelectChange = () => { - if (sourceSelect.value === 'parameter.value') { + if (sourceSelect.value === 'asset.parameter.value') { paramNameSelect.style.display = 'block'; paramNameSelect.style.maxWidth = 'calc(28% - 10px)'; sourceSelect.style.maxWidth = 'calc(25% - 10px)'; @@ -266,7 +274,6 @@ const lookupFFRequest = (app) => { global_id: 'Item Global ID', mpn: 'Item MPN', all: 'Include all', - skip: 'Skip items', }; hideComponent('loader'); @@ -364,7 +371,7 @@ const lookupFFRequest = (app) => { }); document.getElementById('item').addEventListener('change', () => { - if (['skip', 'all'].includes(document.getElementById('item').value)) { + if (document.getElementById('item').value === 'all') { document.getElementById('item_column_group').style.display = 'none'; } else { document.getElementById('item_column_group').style.display = 'block'; @@ -421,7 +428,7 @@ const lookupFFRequest = (app) => { attribute: colSource, }; - if (colSource === 'parameter.value') { + if (colSource === 'asset.parameter.value') { const paramName = document.getElementById(`source-param-select-${index}`).value; columnConfig.parameter_name = paramName; }