Skip to content
This repository was archived by the owner on Nov 8, 2024. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions eppo_client/bandit.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,9 @@
BanditModelData,
BanditNumericAttributeCoefficient,
)
from eppo_client.rules import to_string
from eppo_client.sharders import Sharder
from eppo_client.types import AttributesDict


logger = logging.getLogger(__name__)
Expand All @@ -25,10 +27,42 @@ class Attributes:

@classmethod
def empty(cls):
"""
Create an empty Attributes instance with no numeric or categorical attributes.

Returns:
Attributes: An instance of the Attributes class with empty dictionaries
for numeric and categorical attributes.
"""
return cls({}, {})

@classmethod
def from_dict(cls, attributes: AttributesDict):
"""
Create an Attributes instance from a dictionary of attributes.

Args:
attributes (Dict[str, Union[float, int, bool, str]]): A dictionary where keys are attribute names
and values are attribute values which can be of type float, int, bool, or str.

Returns:
Attributes: An instance of the Attributes class with numeric and categorical attributes separated.
"""
numeric_attributes = {
key: float(value)
for key, value in attributes.items()
if isinstance(value, (int, float))
}
categorical_attributes = {
key: to_string(value)
for key, value in attributes.items()
if isinstance(value, (str, bool))
}
return cls(numeric_attributes, categorical_attributes)


ActionContexts = Dict[str, Attributes]
ActionContextsDict = Dict[str, AttributesDict]


@dataclass
Expand Down
78 changes: 55 additions & 23 deletions eppo_client/client.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,23 @@
import datetime
import logging
import json
from typing import Any, Dict, Optional
from typing import Any, Dict, Optional, Union
from eppo_client.assignment_logger import AssignmentLogger
from eppo_client.bandit import BanditEvaluator, BanditResult, Attributes, ActionContexts
from eppo_client.bandit import (
ActionContextsDict,
BanditEvaluator,
BanditResult,
Attributes,
ActionContexts,
)
from eppo_client.configuration_requestor import (
ExperimentConfigurationRequestor,
)
from eppo_client.constants import POLL_INTERVAL_MILLIS, POLL_JITTER_MILLIS
from eppo_client.models import VariationType
from eppo_client.poller import Poller
from eppo_client.sharders import MD5Sharder
from eppo_client.types import SubjectAttributes, ValueType
from eppo_client.types import AttributesDict, ValueType
from eppo_client.validation import validate_not_blank
from eppo_client.eval import FlagEvaluation, Evaluator, none_result
from eppo_client.version import __version__
Expand Down Expand Up @@ -43,7 +49,7 @@ def get_string_assignment(
self,
flag_key: str,
subject_key: str,
subject_attributes: SubjectAttributes,
subject_attributes: AttributesDict,
default: str,
) -> str:
return self.get_assignment_variation(
Expand All @@ -58,7 +64,7 @@ def get_integer_assignment(
self,
flag_key: str,
subject_key: str,
subject_attributes: SubjectAttributes,
subject_attributes: AttributesDict,
default: int,
) -> int:
return self.get_assignment_variation(
Expand All @@ -73,7 +79,7 @@ def get_numeric_assignment(
self,
flag_key: str,
subject_key: str,
subject_attributes: SubjectAttributes,
subject_attributes: AttributesDict,
default: float,
) -> float:
# convert to float in case we get an int
Expand All @@ -91,7 +97,7 @@ def get_boolean_assignment(
self,
flag_key: str,
subject_key: str,
subject_attributes: SubjectAttributes,
subject_attributes: AttributesDict,
default: bool,
) -> bool:
return self.get_assignment_variation(
Expand All @@ -106,7 +112,7 @@ def get_json_assignment(
self,
flag_key: str,
subject_key: str,
subject_attributes: SubjectAttributes,
subject_attributes: AttributesDict,
default: Dict[Any, Any],
) -> Dict[Any, Any]:
json_value = self.get_assignment_variation(
Expand All @@ -125,7 +131,7 @@ def get_assignment_variation(
self,
flag_key: str,
subject_key: str,
subject_attributes: SubjectAttributes,
subject_attributes: AttributesDict,
default: Optional[ValueType],
expected_variation_type: VariationType,
):
Expand All @@ -149,7 +155,7 @@ def get_assignment_detail(
self,
flag_key: str,
subject_key: str,
subject_attributes: SubjectAttributes,
subject_attributes: AttributesDict,
expected_variation_type: VariationType,
) -> FlagEvaluation:
"""Maps a subject to a variation for a given flag
Expand Down Expand Up @@ -225,8 +231,8 @@ def get_bandit_action(
self,
flag_key: str,
subject_key: str,
subject_context: Attributes,
actions: ActionContexts,
subject_context: Union[Attributes, AttributesDict],
actions: Union[ActionContexts, ActionContextsDict],
default: str,
) -> BanditResult:
"""
Expand All @@ -244,9 +250,11 @@ def get_bandit_action(
Args:
flag_key (str): The feature flag key that contains the bandit as one of the variations.
subject_key (str): The key identifying the subject.
subject_context (Attributes): The subject context
actions (Dict[str, Attributes]): The dictionary that maps action keys
subject_context (Attributes | AttributesDict): The subject context.
If supplying an AttributesDict, it gets converted to an Attributes instance
actions (ActionContexts | ActionContextsDict): The dictionary that maps action keys
to their context of actions with their contexts.
If supplying an AttributesDict, it gets converted to an Attributes instance.
default (str): The default variation to use if the subject is not part of the bandit.

Returns:
Expand All @@ -264,7 +272,8 @@ def get_bandit_action(
categorical_attributes={"country": "USA"}),
{
"action1": Attributes(numeric_attributes={"price": 10.0}, categorical_attributes={"category": "A"}),
"action2": Attributes.empty()
"action2": {"price": 10.0, "category": "B"}
"action3": Attributes.empty(),
},
"default"
)
Expand All @@ -273,7 +282,6 @@ def get_bandit_action(
else:
do_action(result.action)
"""

try:
return self.get_bandit_action_detail(
flag_key,
Expand All @@ -292,14 +300,21 @@ def get_bandit_action_detail(
self,
flag_key: str,
subject_key: str,
subject_context: Attributes,
actions: ActionContexts,
subject_context: Union[Attributes, AttributesDict],
actions: Union[ActionContexts, ActionContextsDict],
default: str,
) -> BanditResult:
subject_attributes = convert_subject_context_to_attributes(subject_context)
action_contexts = convert_actions_to_action_contexts(actions)
Comment on lines +307 to +308
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for helping reduce friction for developers!


# get experiment assignment
# ignoring type because Dict[str, str] satisfies Dict[str, str | ...] but mypy does not understand
variation = self.get_string_assignment(
flag_key, subject_key, subject_context.categorical_attributes, default # type: ignore
flag_key,
subject_key,
subject_attributes.categorical_attributes
| subject_attributes.numeric_attributes, # type: ignore
Comment on lines +315 to +316
Copy link
Collaborator

@rasendubi rasendubi Jun 12, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

minor: one thing that still bothers me is that:

subject_attributes.categorical_attributes | subject_attributes.numeric_attributes != subject_context

It's just seems very hacky and very easy to shoot oneself in the foot with this, as it may introduce subtle evaluation differences. (e.g., get_string_assignment() returns a different value from here when called by the user).

As one example, attributes that are not one of [str,int,float,bool] are silently dropped for bandits. (In normal flag evaluation, they are convertible to string.)

  • We can cover this with a lot of tests
  • Or maybe refactor the code so that we perform similar transformation (float-promotion and converting to string) on all flag evaluation before attributes enter rules evaluation—so that flag evaluation engine only deals with floats and strings.
  • Alternatively, we can make get_string_assignment() accept Union[Attributes, AttributesDict] as well—this way we can pass user-passed attributes unmodified to flag evaluation.
  • At the very least, this deserves a caution comment here.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, this is a great comment!

As one example, attributes that are not one of [str,int,float,bool] are silently dropped for bandits

But the attribute type covers all the accepted cases: AttributeType = Union[str, int, float, bool, None]; Yes None gets dropped but it indicates absence anyway and would evaluate the same way whether we set the value to None or remove the key, value pair altogether.

Perhaps the problem is that we are not dropping none [str,int,float,bool] values when evaluating get_string_assignment?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The issue is that Python's typing is optional, so users are free to pass any type as attribute. It won't give any error and will be handled by applying json.dumps.

So maybe another solution is to be more strict in what attributes we accept. i.e., add a runtime check for attribute type, error or filter out unexpected types.

The issue is also slightly exaggerated for languages that don't have easy union types (e.g., Go accept any value for attributes). Maybe we should handle that in the respective language's implementation but it would be great for the python to set an example here, so this is not copied blindly

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As another inconsistency: ints may loose precision.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not so worried about losing precision -- the loss in precision is orders of magnitude less than the statistical uncertainty in the bandit model

default,
)

# if the variation is not the bandit key, then the subject is not allocated in the bandit
Expand All @@ -318,8 +333,8 @@ def get_bandit_action_detail(
evaluation = self.__bandit_evaluator.evaluate_bandit(
flag_key,
subject_key,
subject_context,
actions,
subject_attributes,
action_contexts,
bandit_data.model_data,
)

Expand All @@ -334,12 +349,12 @@ def get_bandit_action_detail(
"modelVersion": bandit_data.model_version if evaluation else None,
"timestamp": datetime.datetime.utcnow().isoformat(),
"subjectNumericAttributes": (
subject_context.numeric_attributes
subject_attributes.numeric_attributes
if evaluation.subject_attributes
else {}
),
"subjectCategoricalAttributes": (
subject_context.categorical_attributes
subject_attributes.categorical_attributes
if evaluation.subject_attributes
else {}
),
Expand Down Expand Up @@ -410,3 +425,20 @@ def check_value_type_match(
if expected_type == VariationType.BOOLEAN:
return isinstance(value, bool)
return False


def convert_subject_context_to_attributes(
subject_context: Union[Attributes, AttributesDict]
) -> Attributes:
if isinstance(subject_context, dict):
return Attributes.from_dict(subject_context)
return subject_context


def convert_actions_to_action_contexts(
actions: Union[ActionContexts, ActionContextsDict]
) -> ActionContexts:
return {
k: Attributes.from_dict(v) if isinstance(v, dict) else v
for k, v in actions.items()
}
8 changes: 4 additions & 4 deletions eppo_client/eval.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,15 @@
from dataclasses import dataclass
import datetime

from eppo_client.types import SubjectAttributes
from eppo_client.types import AttributesDict


@dataclass
class FlagEvaluation:
flag_key: str
variation_type: VariationType
subject_key: str
subject_attributes: SubjectAttributes
subject_attributes: AttributesDict
allocation_key: Optional[str]
variation: Optional[Variation]
extra_logging: Dict[str, str]
Expand All @@ -28,7 +28,7 @@ def evaluate_flag(
self,
flag: Flag,
subject_key: str,
subject_attributes: SubjectAttributes,
subject_attributes: AttributesDict,
) -> FlagEvaluation:
if not flag.enabled:
return none_result(
Expand Down Expand Up @@ -93,7 +93,7 @@ def none_result(
flag_key: str,
variation_type: VariationType,
subject_key: str,
subject_attributes: SubjectAttributes,
subject_attributes: AttributesDict,
) -> FlagEvaluation:
return FlagEvaluation(
flag_key=flag_key,
Expand Down
6 changes: 3 additions & 3 deletions eppo_client/rules.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import semver

from eppo_client.models import SdkBaseModel
from eppo_client.types import AttributeType, ConditionValueType, SubjectAttributes
from eppo_client.types import AttributeType, ConditionValueType, AttributesDict


class OperatorType(Enum):
Expand All @@ -32,15 +32,15 @@ class Rule(SdkBaseModel):
conditions: List[Condition]


def matches_rule(rule: Rule, subject_attributes: SubjectAttributes) -> bool:
def matches_rule(rule: Rule, subject_attributes: AttributesDict) -> bool:
return all(
evaluate_condition(condition, subject_attributes)
for condition in rule.conditions
)


def evaluate_condition(
condition: Condition, subject_attributes: SubjectAttributes
condition: Condition, subject_attributes: AttributesDict
) -> bool:
subject_value = subject_attributes.get(condition.attribute, None)
if condition.operator == OperatorType.IS_NULL:
Expand Down
2 changes: 1 addition & 1 deletion eppo_client/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,5 @@
ValueType = Union[str, int, float, bool]
AttributeType = Union[str, int, float, bool, None]
ConditionValueType = Union[AttributeType, List[AttributeType]]
SubjectAttributes = Dict[str, AttributeType]
AttributesDict = Dict[str, AttributeType]
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Renamed this to re-use the type

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice, I did the same in Java calling it "EppoAttributes"

Action = str
5 changes: 3 additions & 2 deletions test/client_bandit_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ def init_fixture():
base_url=MOCK_BASE_URL,
api_key="dummy",
assignment_logger=mock_assignment_logger,
is_graceful_mode=False,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oh yeah good catch!

)
)
sleep(0.1) # wait for initialization
Expand All @@ -91,7 +92,7 @@ def test_get_bandit_action_bandit_does_not_exist():
"nonexistent_bandit",
"subject_key",
DEFAULT_SUBJECT_ATTRIBUTES,
[],
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did not catch that this test was broken because is_graceful_mode was set to True

{},
"default_variation",
)
assert result == BanditResult("default_variation", None)
Expand All @@ -100,7 +101,7 @@ def test_get_bandit_action_bandit_does_not_exist():
def test_get_bandit_action_flag_without_bandit():
client = get_instance()
result = client.get_bandit_action(
"a_flag", "subject_key", DEFAULT_SUBJECT_ATTRIBUTES, [], "default_variation"
"a_flag", "subject_key", DEFAULT_SUBJECT_ATTRIBUTES, {}, "default_variation"
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did not catch that this test was broken because is_graceful_mode was set to True

)
assert result == BanditResult("default_variation", None)

Expand Down