diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 83e57db146..6cd8eb2f6a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -60,6 +60,10 @@ Added Contributed by @khushboobhatia01 +* Enhanced 'search' operator to allow complex criteria matching on payload items. #5482 + + Contributed by @erceth + Fixed ~~~~~ diff --git a/st2common/st2common/operators.py b/st2common/st2common/operators.py index 9fb38548f3..f7cf6fc43f 100644 --- a/st2common/st2common/operators.py +++ b/st2common/st2common/operators.py @@ -58,8 +58,10 @@ def search(value, criteria_pattern, criteria_condition, check_function): value: the payload list to search condition: one of: - * any - return true if any items of the list match and false if none of them match - * all - return true if all items of the list match and false if any of them do not match + * any - return true if any payload items of the list match all criteria items + * all - return true if all payload items of the list match all criteria items + * all2any - return true if all payload items of the list match any criteria items + * any2any - return true if any payload items match any criteria items pattern: a dictionary of criteria to apply to each item of the list This operator has O(n) algorithmic complexity in terms of number of child patterns. @@ -86,18 +88,20 @@ def search(value, criteria_pattern, criteria_condition, check_function): ] } - And an example usage in criteria: + Example #1 --- criteria: trigger.fields: type: search # Controls whether this criteria has to match any or all items of the list - condition: any # or all + condition: any # or all or all2any or any2any pattern: # Here our context is each item of the list # All of these patterns have to match the item for the item to match # These are simply other operators applied to each item in the list + # "#" and text after are ignored. + # This allows dictionary keys to be unique but refer to the same field item.field_name: type: "equals" pattern: "Status" @@ -105,59 +109,53 @@ def search(value, criteria_pattern, criteria_condition, check_function): item.to_value: type: "equals" pattern: "Approved" + + item.field_name#1: + type: "greaterthan" + pattern: 40 + + item.field_name#2: + type: "lessthan" + pattern: 50 """ + if isinstance(value, dict): + value = [value] + payloadItemMatch = all + patternMatch = all if criteria_condition == "any": - # Any item of the list can match all patterns - rtn = any( - [ - # Any payload item can match - all( - [ - # Match all patterns - check_function( - child_criterion_k, - child_criterion_v, - PayloadLookup( - child_payload, prefix=TRIGGER_ITEM_PAYLOAD_PREFIX - ), - ) - for child_criterion_k, child_criterion_v in six.iteritems( - criteria_pattern - ) - ] - ) - for child_payload in value - ] - ) - elif criteria_condition == "all": - # Every item of the list must match all patterns - rtn = all( - [ - # All payload items must match - all( - [ - # Match all patterns - check_function( - child_criterion_k, - child_criterion_v, - PayloadLookup( - child_payload, prefix=TRIGGER_ITEM_PAYLOAD_PREFIX - ), - ) - for child_criterion_k, child_criterion_v in six.iteritems( - criteria_pattern - ) - ] - ) - for child_payload in value - ] - ) - else: + payloadItemMatch = any + elif criteria_condition == "all2any": + patternMatch = any + elif criteria_condition == "any2any": + payloadItemMatch = any + patternMatch = any + elif criteria_condition != "all": raise UnrecognizedConditionError( - "The '%s' search condition is not recognized, only 'any' " - "and 'all' are allowed" % criteria_condition + "The '%s' condition is not recognized for type search, 'any', 'all', 'any2any'" + " and 'all2any' are allowed" % criteria_condition ) + rtn = payloadItemMatch( + [ + # any/all payload item can match + patternMatch( + [ + # Match any/all patterns + check_function( + child_criterion_k, + child_criterion_v, + PayloadLookup( + child_payload, prefix=TRIGGER_ITEM_PAYLOAD_PREFIX + ), + ) + for child_criterion_k, child_criterion_v in six.iteritems( + criteria_pattern + ) + ] + ) + for child_payload in value + ] + ) return rtn diff --git a/st2common/tests/unit/test_operators.py b/st2common/tests/unit/test_operators.py index 7198f26fb1..dd00eba6f7 100644 --- a/st2common/tests/unit/test_operators.py +++ b/st2common/tests/unit/test_operators.py @@ -564,6 +564,203 @@ def record_function_args(criterion_k, criterion_v, payload_lookup): ], ) + def _test_function(self, criterion_k, criterion_v, payload_lookup): + op = operators.get_operator(criterion_v["type"]) + return op(payload_lookup.get_value("item.to_value")[0], criterion_v["pattern"]) + + def test_search_any2any(self): + # true if any payload items match any criteria + op = operators.get_operator("search") + + payload = [ + { + "field_name": "waterLevel", + "to_value": 30, + }, + { + "field_name": "waterLevel", + "to_value": 45, + }, + ] + + criteria_pattern = { + "item.waterLevel#1": { + "type": "lessthan", + "pattern": 40, + }, + "item.waterLevel#2": { + "type": "greaterthan", + "pattern": 50, + }, + } + + result = op(payload, criteria_pattern, "any2any", self._test_function) + self.assertTrue(result) + + payload[0]["to_value"] = 44 + + result = op(payload, criteria_pattern, "any2any", self._test_function) + self.assertFalse(result) + + def test_search_any(self): + # true if any payload items match all criteria + op = operators.get_operator("search") + payload = [ + { + "field_name": "waterLevel", + "to_value": 45, + }, + { + "field_name": "waterLevel", + "to_value": 20, + }, + ] + + criteria_pattern = { + "item.waterLevel#1": { + "type": "greaterthan", + "pattern": 40, + }, + "item.waterLevel#2": { + "type": "lessthan", + "pattern": 50, + }, + "item.waterLevel#3": { + "type": "equals", + "pattern": 46, + }, + } + + result = op(payload, criteria_pattern, "any", self._test_function) + self.assertFalse(result) + + payload[0]["to_value"] = 46 + + result = op(payload, criteria_pattern, "any", self._test_function) + self.assertTrue(result) + + payload[0]["to_value"] = 45 + del criteria_pattern["item.waterLevel#3"] + + result = op(payload, criteria_pattern, "any", self._test_function) + self.assertTrue(result) + + def test_search_all2any(self): + # true if all payload items match any criteria + op = operators.get_operator("search") + payload = [ + { + "field_name": "waterLevel", + "to_value": 45, + }, + { + "field_name": "waterLevel", + "to_value": 20, + }, + ] + + criteria_pattern = { + "item.waterLevel#1": { + "type": "greaterthan", + "pattern": 40, + }, + "item.waterLevel#2": { + "type": "lessthan", + "pattern": 50, + }, + "item.waterLevel#3": { + "type": "equals", + "pattern": 46, + }, + } + + result = op(payload, criteria_pattern, "all2any", self._test_function) + self.assertTrue(result) + + criteria_pattern["item.waterLevel#2"]["type"] = "greaterthan" + + result = op(payload, criteria_pattern, "all2any", self._test_function) + self.assertFalse(result) + + def test_search_all(self): + # true if all payload items match all criteria items + op = operators.get_operator("search") + payload = [ + { + "field_name": "waterLevel", + "to_value": 45, + }, + { + "field_name": "waterLevel", + "to_value": 46, + }, + ] + + criteria_pattern = { + "item.waterLevel#1": { + "type": "greaterthan", + "pattern": 40, + }, + "item.waterLevel#2": { + "type": "lessthan", + "pattern": 50, + }, + } + + result = op(payload, criteria_pattern, "all", self._test_function) + self.assertTrue(result) + + payload[0]["to_value"] = 30 + + result = op(payload, criteria_pattern, "all", self._test_function) + self.assertFalse(result) + + payload[0]["to_value"] = 45 + + criteria_pattern["item.waterLevel#3"] = { + "type": "equals", + "pattern": 46, + } + + result = op(payload, criteria_pattern, "all", self._test_function) + self.assertFalse(result) + + def test_search_payload_dict(self): + op = operators.get_operator("search") + payload = { + "field_name": "waterLevel", + "to_value": 45, + } + + criteria_pattern = { + "item.waterLevel#1": { + "type": "greaterthan", + "pattern": 40, + }, + "item.waterLevel#2": { + "type": "lessthan", + "pattern": 50, + }, + } + + result = op(payload, criteria_pattern, "all", self._test_function) + self.assertTrue(result) + + payload["to_value"] = 30 + + result = op(payload, criteria_pattern, "all", self._test_function) + self.assertFalse(result) + + payload["to_value"] = 45 + + criteria_pattern["item.waterLevel#3"] = { + "type": "equals", + "pattern": 46, + } + + result = op(payload, criteria_pattern, "all", self._test_function) + self.assertFalse(result) + class OperatorTest(unittest2.TestCase): def test_matchwildcard(self): diff --git a/st2reactor/st2reactor/rules/filter.py b/st2reactor/st2reactor/rules/filter.py index 838adc5480..f2b4c7b753 100644 --- a/st2reactor/st2reactor/rules/filter.py +++ b/st2reactor/st2reactor/rules/filter.py @@ -154,8 +154,10 @@ def _check_criterion(self, criterion_k, criterion_v, payload_lookup): return (False, None, None) + # Avoids the dict unique keys limitation. Allows multiple evaluations of the same payload item by a rule. + criterion_k_hash_strip = criterion_k.split("#", 1)[0] try: - matches = payload_lookup.get_value(criterion_k) + matches = payload_lookup.get_value(criterion_k_hash_strip) # pick value if only 1 matches else will end up being an array match. if matches: payload_value = matches[0] if len(matches) > 0 else matches diff --git a/st2reactor/tests/unit/test_filter.py b/st2reactor/tests/unit/test_filter.py index d1e42eaece..fd7c9d5741 100644 --- a/st2reactor/tests/unit/test_filter.py +++ b/st2reactor/tests/unit/test_filter.py @@ -414,3 +414,25 @@ class MockSystemLookup(object): } f = RuleFilter(MOCK_TRIGGER_INSTANCE, MOCK_TRIGGER, rule) self.assertTrue(f.filter()) + + def test_hash_strip_int_value(self): + rule = MOCK_RULE_1 + rule.criteria = { + "trigger.int": {"type": "gt", "pattern": 0}, + "trigger.int#2": {"type": "lt", "pattern": 2}, + } + f = RuleFilter(MOCK_TRIGGER_INSTANCE, MOCK_TRIGGER, rule) + self.assertTrue(f.filter(), "equals check should have passed.") + + rule = MOCK_RULE_1 + rule.criteria = { + "trigger.int": {"type": "gt", "pattern": 2}, + "trigger.int#2": {"type": "lt", "pattern": 3}, + } + f = RuleFilter(MOCK_TRIGGER_INSTANCE, MOCK_TRIGGER, rule) + self.assertFalse(f.filter(), "trigger value is gt than 0 but didn't match.") + + rule = MOCK_RULE_1 + rule.criteria = {"trigger.int#1": {"type": "lt", "pattern": 2}} + f = RuleFilter(MOCK_TRIGGER_INSTANCE, MOCK_TRIGGER, rule) + self.assertTrue(f.filter(), "trigger value is gt than 0 but didn't match.")