From 74f690a261bcb45dc91936bc60d4785fdf3e526f Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 12:05:23 -0400 Subject: [PATCH 001/142] Update code_object_pickler.py to include consistent pickling for lambdas --- .../internal/code_object_pickler.py | 388 ++++++++++++++++++ 1 file changed, 388 insertions(+) diff --git a/sdks/python/apache_beam/internal/code_object_pickler.py b/sdks/python/apache_beam/internal/code_object_pickler.py index c3658120b4ef..e38a3b740156 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler.py +++ b/sdks/python/apache_beam/internal/code_object_pickler.py @@ -19,3 +19,391 @@ def get_normalized_path(path): """Returns a normalized path. This function is intended to be overridden.""" return path + +def _get_code_path(callable): + """Returns the stable reference to the code object. + + Args: + callable: The callable object to search for. + + Returns: + The stable reference to the code object. + """ + code_path = _extend_path( + callable.__module__, + _search( + callable, + sys.modules[callable.__module__], + callable.__qualname__.split('.'), + ), + ) + return code_path + + +def _extend_path(prefix, suffix): + """Extends the path to the code object. + + Args: + prefix: The prefix of the path. + suffix: The rest of the path. + + Returns: + The extended path. + """ + if suffix is None: + return None + if not suffix: + return prefix + return prefix + '.' + suffix + + +def _search(callable, node, qual_name_parts): + """Searches an object to create a stable reference code path. + + Recursively searches the tree of objects starting from node and looking for + callable and returns a string to uniquely identify the path from node to the + callable. + + Args: + callable: The callable object to search for. + node: The object to search within. + qual_name_parts: A list of strings representing the qualified name. + + Returns: + The stable reference to the code object, or None if not found. + """ + if node is None: + return None + if not qual_name_parts: + if hasattr(node, '__code__') and node.__code__ == callable.__code__: + return '__code__' + else: + return None + if inspect.ismodule(node) or inspect.isclass(node): + return _search_module_or_class(callable, node, qual_name_parts) + elif inspect.isfunction(node): + return _search_function(callable, node, qual_name_parts) + elif inspect.iscode(node): + return _search_code(callable, node, qual_name_parts) + + +def _search_module_or_class(callable, node, qual_name_parts): + """Searches a module or class to create a stable reference code path. + + Args: + callable: The callable object to search for. + node: The module or class to search within. + qual_name_parts: The list of qual name parts. + + Returns: + The stable reference to the code object, or None if not found. + """ + # Functions/methods have a name that is unique within a given module or class + # so the traversal can directly lookup function object identified by the name. + # Lambdas don't have a name so we need to search all the attributes of the + # node. + first_part = qual_name_parts[0] + rest = qual_name_parts[1:] + if first_part == '': + for name in dir(node): + value = getattr(node, name) + if (isinstance(value, type(callable)) + and value.__code__ == callable.__code__): + return name + '.__code__' + elif (isinstance(value, types.FunctionType) + and value.__defaults__ is not None): + # Python functions can have other functions as default parameters which + # might contain the code object so we have to search them. + for i, default_param_value in enumerate(value.__defaults__): + path = _search(callable, default_param_value, rest) + if path is not None: + return _extend_path(name, _extend_path(f'__defaults__[{i}]', path)) + else: + return _extend_path( + first_part, _search(callable, getattr(node, first_part), rest) + ) + + +def _search_function(callable, node, qual_name_parts): + """Searches a function to create a stable reference code path. + + Args: + callable: The callable object to search for. + node: The function to search within. + qual_name_parts: The list of qual name parts. + + Returns: + The stable reference to the code object, or None if not found. + """ + first_part = qual_name_parts[0] + if node.__code__ == callable.__code__: + if len(qual_name_parts) > 1: + raise ValueError('Qual name parts too long') + return '__code__' + # If first part is '' then the code object is in a local variable + # so we should add __code__ to the path to indicate that we are entering + # the code object of the function. + if first_part == '': + return _extend_path( + '__code__', _search(callable, node.__code__, qual_name_parts) + ) + + +def _search_code(callable, node, qual_name_parts): + """Searches a code object to create a stable reference code path. + + Args: + callable: The callable to search for. + node: The code object to search within. + qual_name_parts: The list of qual name parts. + + Returns: + The stable reference to the code object, or None if not found. + + Raises: + ValueError: If the qual name parts are too long. + """ + first_part = qual_name_parts[0] + rest = qual_name_parts[1:] + if node == callable.__code__: + if len(qual_name_parts) > 1: + raise ValueError('Qual name parts too long') + return '' + elif first_part == '': + code_objects_by_name = collections.defaultdict(list) + for co_const in node.co_consts: + if inspect.iscode(co_const): + code_objects_by_name[co_const.co_name].append(co_const) + num_lambdas = len(code_objects_by_name.get('', [])) + # If there is only one lambda, we can use the default path + # 'co_consts[]'. This is the most common case and it is + # faster than calculating the signature and the hash. + if num_lambdas == 1: + path = _search(callable, code_objects_by_name[''][0], rest) + if path is not None: + return _extend_path('co_consts[]', path) + else: + return _search_lambda(callable, code_objects_by_name, rest) + elif node.co_name == first_part: + return _search(callable, node, rest) + + +def _search_lambda(callable, code_objects_by_name, qual_name_parts): + """Searches a lambda to create a stable reference code path. + + Args: + callable: The callable to search for. + code_objects_by_name: The code objects to search within, keyed by name. + qual_name_parts: The rest of the qual_name_parts. + + Returns: + The stable reference to the code object, or None if not found. + """ + # There are multiple lambdas in the code object, so we need to calculate + # the signature and the hash to identify the correct lambda. + lambda_code_objects_by_name = collections.defaultdict(list) + name = qual_name_parts[0] + code_objects = code_objects_by_name[name] + if name == '': + for code_object in code_objects: + lambda_name = f', {_signature(code_object)}' + lambda_code_objects_by_name[lambda_name].append(code_object) + # Check if there are any lambdas with the same signature. + # If there are, we need to calculate the hash to identify the correct + # lambda. + for lambda_name, lambda_objects in lambda_code_objects_by_name.items(): + if len(lambda_objects) > 1: + for lambda_object in lambda_objects: + path = _search(callable, lambda_object, qual_name_parts) + if path is not None: + return _extend_path( + f'co_consts[{lambda_name},' + f' {_create_bytecode_hash(lambda_object)}]', + path, + ) + else: + # If there is only one lambda with this signature, we can + # use the signature to identify the correct lambda. + path = _search(callable, code_objects[0], qual_name_parts) + if path is not None: + return _extend_path(f'co_consts[{lambda_name}]', path) + else: + # For non lambda objects, we can use the name to identify the object. + path = _search(callable, code_objects[0], qual_name_parts) + if path is not None: + return _extend_path(f'co_consts[{name}]', path) + + +# Matches a path like: co_consts[my_function] +_SINGLE_NAME_PATTERN = re.compile(r'co_consts\[([a-zA-Z0-9\<\>_-]+)]') +# Matches a path like: co_consts[, ('x',)] +_LAMBDA_WITH_ARGS_PATTERN = re.compile( + r"co_consts\[(<[^>]+>),\s*(\('[^']*'\s*,\s*\))\]" +) +# Matches a path like: co_consts[, ('x',), 1234567890] +_LAMBDA_WITH_HASH_PATTERN = re.compile( + r"co_consts\[(<[^>]+>),\s*(\('[^']*'\s*,\s*\)),\s*(.+)\]" +) +# Matches a path like: __defaults__[0] +_DEFAULT_PATTERN = re.compile(r'(__defaults__)\[(\d+)\]') +# Matches an argument like: 'x' +_ARGUMENT_PATTERN = re.compile(r"'([^']*)'") + + +def _get_code_object_from_single_name_pattern(obj, name_result, path): + """Returns the code object from a name pattern. + + Args: + obj: The object to search within. + name_result: The result of the name pattern search. + path: The path to the code object. + + Returns: + The code object. + + Raises: + ValueError: If the pattern is invalid. + """ + if len(name_result.groups()) > 1: + raise ValueError(f'Invalid pattern for single name: {name_result.group(0)}') + # Groups are indexed starting at 1, group(0) is the entire match. + name = name_result.group(1) + for co_const in obj.co_consts: + if inspect.iscode(co_const) and co_const.co_name == name: + return co_const + raise AttributeError(f'Could not find code object with path: {path}') + + +def _get_code_object_from_lambda_with_args_pattern( + obj, lambda_with_args_result, path +): + """Returns the code object from a lambda with args pattern. + + Args: + obj: The object to search within. + lambda_with_args_result: The result of the lambda with args pattern search. + path: The path to the code object. + + Returns: + The code object. + """ + name = lambda_with_args_result.group(1) + code_objects = collections.defaultdict(list) + for co_const in obj.co_consts: + if inspect.iscode(co_const) and co_const.co_name == name: + code_objects[co_const.co_name].append(co_const) + for name, objects in code_objects.items(): + for obj_ in objects: + args = tuple( + re.findall(_ARGUMENT_PATTERN, lambda_with_args_result.group(2)) + ) + if obj_.co_varnames == args: + return obj_ + raise AttributeError(f'Could not find code object with path: {path}') + + +def _get_code_object_from_lambda_with_hash_pattern( + obj, lambda_with_hash_result, path +): + """Returns the code object from a lambda with hash pattern. + + Args: + obj: The object to search within. + lambda_with_hash_result: The result of the lambda with hash pattern search. + path: The path to the code object. + + Returns: + The code object. + """ + name = lambda_with_hash_result.group(1) + code_objects = collections.defaultdict(list) + for co_const in obj.co_consts: + if inspect.iscode(co_const) and co_const.co_name == name: + code_objects[co_const.co_name].append(co_const) + for name, objects in code_objects.items(): + for obj_ in objects: + args = tuple( + re.findall(_ARGUMENT_PATTERN, lambda_with_hash_result.group(2)) + ) + if obj_.co_varnames == args: + hash_value = lambda_with_hash_result.group(3) + if hash_value == str(_create_bytecode_hash(obj_)): + return obj_ + raise AttributeError(f'Could not find code object with path: {path}') + + +def _get_code_from_stable_reference(path): + """Returns the code object from a stable reference. + + Args: + path: A string representing the stable reference to the code object. + + Returns: + The code object. + + Raises: + ValueError: If the path is empty or invalid. + AttributeError: If the attribute is not found. + """ + if not path: + raise ValueError('Path must not be empty.') + parts = path.split('.') + obj = sys.modules[parts[0]] + for part in parts[1:]: + if name_result := _SINGLE_NAME_PATTERN.fullmatch(part): + obj = _get_code_object_from_single_name_pattern(obj, name_result, path) + elif lambda_with_args_result := _LAMBDA_WITH_ARGS_PATTERN.fullmatch(part): + obj = _get_code_object_from_lambda_with_args_pattern( + obj, lambda_with_args_result, path + ) + elif lambda_with_hash_result := _LAMBDA_WITH_HASH_PATTERN.fullmatch(part): + obj = _get_code_object_from_lambda_with_hash_pattern( + obj, lambda_with_hash_result, path + ) + elif default_result := _DEFAULT_PATTERN.fullmatch(part): + index = int(default_result.group(2)) + if index >= len(obj.__defaults__): + raise ValueError( + f'Index {index} is out of bounds for obj.__defaults__' + f' {len(obj.__defaults__)} in path {path}' + ) + obj = getattr(obj, '__defaults__')[index] + else: + obj = getattr(obj, part) + return obj + + +def _signature(obj): + """Returns the signature of a code object. + + The signature is the names of the arguments of the code object. This is used + to unique identify lambdas. + + Args: + obj: A code object, function, method, or cell. + + Returns: + A tuple of the names of the arguments of the code object. + """ + if isinstance(obj, types.CodeType): + arg_count = ( + obj.co_argcount + + obj.co_kwonlyargcount + + (obj.co_flags & 4 == 4) # PyCF_VARARGS + + (obj.co_flags & 8 == 8) # PyCF_VARKEYWORDS + ) + return obj.co_varnames[:arg_count] + else: + return None + + +def _create_bytecode_hash(code_object): + """Returns the hash of a code object. + + Args: + code_object: A code object. + + Returns: + The hash of the code object. + """ + return hashlib.md5(code_object.co_code).hexdigest() From 23fa4d8a310c4db3bc2d717bfa0e22acfe3baf90 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 12:14:07 -0400 Subject: [PATCH 002/142] Create code_object_pickler_test.py --- .../internal/code_object_pickler_test.py | 155 ++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 sdks/python/apache_beam/internal/code_object_pickler_test.py diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py new file mode 100644 index 000000000000..4ce8488722e2 --- /dev/null +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -0,0 +1,155 @@ +from apache_beam.internal import code_object_pickler + +from google3.testing.pybase import googletest +from google3.testing.pybase import parameterized + + +def top_level_function(): + return 1 + + +top_level_lambda = lambda x: 1 + + +def get_nested_function(): + def nested_function(): + return 1 + return nested_function + + +def get_lambda_from_dictionary(): + d = { + "a": lambda x: 1, + "b": lambda y: 2 + } + return d["a"] + + +def get_lambda_from_dictionary_same_args(): + d = { + "a": lambda x: 1, + "b": lambda x: x + 1 + } + return d["a"] + + +def function_with_lambda_default_argument(fn=lambda x: 1): + return fn + + +def function_with_function_default_argument(fn=top_level_function): + return fn + + +def function_decorator(f): + return lambda x: f(f(x)) + + +@function_decorator +def add_one(x): + return x + 1 + + +class ClassWithFunction: + def process(self): + return 1 + + +class ClassWithStaticMethod: + @staticmethod + def static_method(): # pylint: disable=staticmethod-use + return 1 + + +class ClassWithClassMethod: + @classmethod + def class_method(cls): + return 1 + + +class ClassWithNestedFunction: + def process(self): + def nested_function(): + return 1 + return nested_function + + +class ClassWithLambda: + def process(self): + return lambda: 1 + + +class ClassWithNestedClass: + class InnerClass: + def process(self): + return 1 + + +class ClassWithNestedLambda: + def process(self): + def get_lambda_from_dictionary(): + d = { + "a": lambda x: 1, + "b": lambda y: 2 + } + return d["a"] + return get_lambda_from_dictionary() + +test_cases = [ + (top_level_function, "__main__.top_level_function.__code__"), + (top_level_lambda, "__main__.top_level_lambda.__code__"), + (get_nested_function(), + ("__main__.get_nested_function.__code__.co_consts" + "[nested_function]")), + (get_lambda_from_dictionary(), + ("__main__.get_lambda_from_dictionary.__code__.co_consts" + "[, ('x',)]")), + (get_lambda_from_dictionary_same_args(), + ("__main__.get_lambda_from_dictionary_same_args.__code__.co_consts" + "[, ('x',), " + "ac455edc80140a7f12ad229519456f71]")), + (function_with_lambda_default_argument(), + ("__main__.function_with_lambda_default_argument.__defaults__[0]" + ".__code__")), + (function_with_function_default_argument(), + "__main__.top_level_function.__code__"), + (add_one, + "__main__.function_decorator.__code__.co_consts[]"), + (ClassWithFunction.process, + "__main__.ClassWithFunction.process.__code__"), + (ClassWithStaticMethod.static_method, + "__main__.ClassWithStaticMethod.static_method.__code__"), + (ClassWithClassMethod.class_method, + "__main__.ClassWithClassMethod.class_method.__code__"), + (ClassWithNestedFunction().process(), + ("__main__.ClassWithNestedFunction.process.__code__.co_consts" + "[nested_function]")), + (ClassWithLambda().process(), + "__main__.ClassWithLambda.process.__code__.co_consts[]"), + (ClassWithNestedClass.InnerClass().process, + "__main__.ClassWithNestedClass.InnerClass.process.__code__"), + (ClassWithNestedLambda().process(), + ("__main__.ClassWithNestedLambda.process.__code__.co_consts" + "[get_lambda_from_dictionary].co_consts[, ('x',)]")), + (ClassWithNestedLambda.process, + "__main__.ClassWithNestedLambda.process.__code__"), + ] + + +class DillTest(parameterized.TestCase): + + @parameterized.parameters(test_cases) + def test_get_code_path(self, callable, expected): + actual = code_object_pickler._get_code_path(callable) + self.assertEqual(actual, expected) + + @parameterized.parameters(test_cases) + def test_get_code_from_stable_reference(self, callable, path): + actual = code_object_pickler._get_code_from_stable_reference(path) + self.assertEqual(actual, callable.__code__) + + @parameterized.parameters(test_cases) + def test_roundtrip(self, callable, _): + path = code_object_pickler._get_code_path(callable) + actual = code_object_pickler._get_code_from_stable_reference(path) + self.assertEqual(actual, callable.__code__) From 86e12b2f15cbcd709501e0ae8a97a93d79bd23c0 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 13:12:59 -0400 Subject: [PATCH 003/142] Update code_object_pickler_test.py --- .../apache_beam/internal/code_object_pickler_test.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index 4ce8488722e2..be5173a6f09f 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -1,6 +1,6 @@ -from apache_beam.internal import code_object_pickler +import unittest -from google3.testing.pybase import googletest +from apache_beam.internal import code_object_pickler from google3.testing.pybase import parameterized @@ -153,3 +153,7 @@ def test_roundtrip(self, callable, _): path = code_object_pickler._get_code_path(callable) actual = code_object_pickler._get_code_from_stable_reference(path) self.assertEqual(actual, callable.__code__) + +if __name__ == "__main__": + unittest.main() + From d17d08a5bf9d856a456af95f13f8ff42cd1c1196 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 13:25:13 -0400 Subject: [PATCH 004/142] Create after_module_add_function.py --- .../test_cases/after_module_add_function.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 sdks/python/apache_beam/internal/test_cases/after_module_add_function.py diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py b/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py new file mode 100644 index 000000000000..06ed3283b78f --- /dev/null +++ b/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py @@ -0,0 +1,13 @@ +"""Module for testing code path generation with functions.""" + + +def another_function(): + a = 1 + b = lambda: 2 + return b + + +def my_function(): + a = 1 + b = lambda: 2 + return b From 6b66de9548c2fb2de5ae09d4baa8e9af0dbd2307 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 13:26:06 -0400 Subject: [PATCH 005/142] Create after_module_add_lambda_variable.py --- .../test_cases/after_module_add_lambda_variable.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 sdks/python/apache_beam/internal/test_cases/after_module_add_lambda_variable.py diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_add_lambda_variable.py b/sdks/python/apache_beam/internal/test_cases/after_module_add_lambda_variable.py new file mode 100644 index 000000000000..205fa64ceaed --- /dev/null +++ b/sdks/python/apache_beam/internal/test_cases/after_module_add_lambda_variable.py @@ -0,0 +1,7 @@ +"""Module for testing code path generation with functions.""" + +def my_function(): + a = 1 + b = lambda: 2 + new_lambda_variable = lambda: 4 + return b From a1106e6454e709d6d3b2ac667b3572ee86ea64fe Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 13:26:53 -0400 Subject: [PATCH 006/142] Create after_module_add_variable.py --- .../internal/test_cases/after_module_add_variable.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 sdks/python/apache_beam/internal/test_cases/after_module_add_variable.py diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_add_variable.py b/sdks/python/apache_beam/internal/test_cases/after_module_add_variable.py new file mode 100644 index 000000000000..788b988c580a --- /dev/null +++ b/sdks/python/apache_beam/internal/test_cases/after_module_add_variable.py @@ -0,0 +1,8 @@ +"""Module for testing code path generation with functions.""" + + +def my_function(): + a = 1 + b = lambda: 2 + new_local_variable = 3 + return b From bbe555c178765e1332d703a1f726a4ae40e416d3 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 13:27:37 -0400 Subject: [PATCH 007/142] Create after_module_remove_lambda_variable.py --- .../test_cases/after_module_remove_lambda_variable.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 sdks/python/apache_beam/internal/test_cases/after_module_remove_lambda_variable.py diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_remove_lambda_variable.py b/sdks/python/apache_beam/internal/test_cases/after_module_remove_lambda_variable.py new file mode 100644 index 000000000000..b24f5bfc2643 --- /dev/null +++ b/sdks/python/apache_beam/internal/test_cases/after_module_remove_lambda_variable.py @@ -0,0 +1,6 @@ +"""Module for testing code path generation with functions.""" + + +def my_function(): + b = lambda: 2 + return b From 4a8d35007623b71500d80e19c4bb61c657713a39 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 13:28:10 -0400 Subject: [PATCH 008/142] Create after_module_remove_variable.py --- .../internal/test_cases/after_module_remove_variable.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 sdks/python/apache_beam/internal/test_cases/after_module_remove_variable.py diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_remove_variable.py b/sdks/python/apache_beam/internal/test_cases/after_module_remove_variable.py new file mode 100644 index 000000000000..b24f5bfc2643 --- /dev/null +++ b/sdks/python/apache_beam/internal/test_cases/after_module_remove_variable.py @@ -0,0 +1,6 @@ +"""Module for testing code path generation with functions.""" + + +def my_function(): + b = lambda: 2 + return b From 3941819daa15616bc60e104b0a0db7a3bebf0b40 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 13:28:57 -0400 Subject: [PATCH 009/142] Create after_module_with_classes.py --- .../test_cases/after_module_with_classes.py | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py new file mode 100644 index 000000000000..20f4e6a59a64 --- /dev/null +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py @@ -0,0 +1,69 @@ +"""Module for testing code path generation with classes.""" + + +class AddLocalVariable: + def my_method(): + a = 1 + b = lambda: 2 + new_local_variable = 3 + return b + + +class RemoveLocalVariable: + def my_method(): + b = lambda: 2 + return b + + +class AddLambdaVariable: + def my_method(): + a = 1 + b = lambda: 2 + c = lambda: 3 + return b + + +class RemoveLambdaVariable: + def my_method(): + b = lambda: 2 + return b + + +class ClassWithNestedFunction: + def my_method(): + def nested_function(): + c = 3 + return c + a = 1 + b = lambda: 2 + return b + + +class ClassWithNestedFunction2: + def my_method(): + a = 1 + b = lambda: 2 + def nested_function(): + c = 3 + return c + return b + + +class ClassWithTwoMethods: + def another_method(): + a = 1 + b = lambda: 2 + return b + + def my_method(): + a = 1 + b = lambda: 2 + return b + + +class RemoveMethod: + def my_method(): + a = 1 + + b = lambda: 2 + return b From c2ab7134ca52efadececc3349ec9f63964d02789 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 13:30:56 -0400 Subject: [PATCH 010/142] Create after_module_with_global_variable.py --- .../test_cases/after_module_with_global_variable.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 sdks/python/apache_beam/internal/test_cases/after_module_with_global_variable.py diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_global_variable.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_global_variable.py new file mode 100644 index 000000000000..d54f76626291 --- /dev/null +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_global_variable.py @@ -0,0 +1,9 @@ +"""Module for testing code path generation with a global variable.""" + +GLOBAL_VARIABLE = lambda: 3 + + +def my_function(): + a = 1 + b = lambda: 2 + return b From ca853c29a07c7559ceac6cbd5114c64a18ecd56a Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 13:31:33 -0400 Subject: [PATCH 011/142] Create after_module_with_nested_function.py --- .../test_cases/after_module_with_nested_function.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py new file mode 100644 index 000000000000..1f1c3ad9a9ab --- /dev/null +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py @@ -0,0 +1,11 @@ +"""Module for testing code path generation with functions.""" + + +def my_function(): + a = 1 + b = lambda: 2 + def nested_function(): + c = 3 + return c + return b + From 8c26a6c3d031b2ba4ec1dbdb55ba2bb44032efa2 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 13:32:13 -0400 Subject: [PATCH 012/142] Create after_module_with_nested_function_2.py --- .../test_cases/after_module_with_nested_function_2.py | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function_2.py diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function_2.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function_2.py new file mode 100644 index 000000000000..8de902e559d0 --- /dev/null +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function_2.py @@ -0,0 +1,10 @@ +"""Module for testing code path generation with functions.""" + + +def my_function(): + def nested_function(): + c = 3 + return c + a = 1 + b = lambda: 2 + return b From 40b767c3b1168035aa3147819d1028d1afbf962c Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 13:32:46 -0400 Subject: [PATCH 013/142] Create after_module_with_single_class.py --- .../test_cases/after_module_with_single_class.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 sdks/python/apache_beam/internal/test_cases/after_module_with_single_class.py diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_single_class.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_single_class.py new file mode 100644 index 000000000000..79151f16373e --- /dev/null +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_single_class.py @@ -0,0 +1,14 @@ +"""Module for testing code path generation with a single class.""" + + +class MyClass: + def another_function(): + a = 1 + b = lambda: 2 + return b + + +def my_function(): + a = 1 + b = lambda: 2 + return b From 251b1d7e8c555918c60420ba80251fab41c319a8 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 13:33:19 -0400 Subject: [PATCH 014/142] Create before_module_with_classes.py --- .../test_cases/before_module_with_classes.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 sdks/python/apache_beam/internal/test_cases/before_module_with_classes.py diff --git a/sdks/python/apache_beam/internal/test_cases/before_module_with_classes.py b/sdks/python/apache_beam/internal/test_cases/before_module_with_classes.py new file mode 100644 index 000000000000..4f3b8fb2e63c --- /dev/null +++ b/sdks/python/apache_beam/internal/test_cases/before_module_with_classes.py @@ -0,0 +1,62 @@ +"""Module for testing code path generation with classes.""" + + +class AddLocalVariable: + def my_method(): + a = 1 + b = lambda: 2 + return b + + +class RemoveLocalVariable: + def my_method(): + a = 1 + b = lambda: 2 + return b + + +class AddLambdaVariable: + def my_method(): + a = 1 + b = lambda: 2 + return b + + +class RemoveLambdaVariable: + def my_method(): + a = lambda: 1 + b = lambda: 2 + return b + + +class ClassWithNestedFunction: + def my_method(): + a = 1 + b = lambda: 2 + return b + + +class ClassWithNestedFunction2: + def my_method(): + a = 1 + b = lambda: 2 + return b + + +class ClassWithTwoMethods: + def my_method(): + a = 1 + b = lambda: 2 + return b + + +class RemoveMethod: + def another_method(): + a = 1 + b = lambda: 2 + return b + + def my_method(): + a = 1 + b = lambda: 2 + return b From 3590211de55b6c99cff3066c24ced48897a5e6b0 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 13:34:47 -0400 Subject: [PATCH 015/142] Create before_module_with_functions.py --- .../internal/test_cases/before_module_with_functions.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 sdks/python/apache_beam/internal/test_cases/before_module_with_functions.py diff --git a/sdks/python/apache_beam/internal/test_cases/before_module_with_functions.py b/sdks/python/apache_beam/internal/test_cases/before_module_with_functions.py new file mode 100644 index 000000000000..34614ca51d7c --- /dev/null +++ b/sdks/python/apache_beam/internal/test_cases/before_module_with_functions.py @@ -0,0 +1,7 @@ +"""Module for testing code path generation with functions.""" + + +def my_function(): + a = 1 + b = lambda: 2 + return b From 6169dbefbfee632b2da0c4c5033ba2da4bfbedd1 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 13:35:14 -0400 Subject: [PATCH 016/142] Create before_module_with_lambdas.py --- .../internal/test_cases/before_module_with_lambdas.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 sdks/python/apache_beam/internal/test_cases/before_module_with_lambdas.py diff --git a/sdks/python/apache_beam/internal/test_cases/before_module_with_lambdas.py b/sdks/python/apache_beam/internal/test_cases/before_module_with_lambdas.py new file mode 100644 index 000000000000..5265c4ce3c71 --- /dev/null +++ b/sdks/python/apache_beam/internal/test_cases/before_module_with_lambdas.py @@ -0,0 +1,7 @@ +"""Module for testing code path generation with functions.""" + + +def my_function(): + a = lambda: 1 + b = lambda: 2 + return b From 03ba2587d6d4aa37bb47b9c0d6e482e2936af887 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 13:35:43 -0400 Subject: [PATCH 017/142] Create module_with_default_argument.py --- .../internal/test_cases/module_with_default_argument.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 sdks/python/apache_beam/internal/test_cases/module_with_default_argument.py diff --git a/sdks/python/apache_beam/internal/test_cases/module_with_default_argument.py b/sdks/python/apache_beam/internal/test_cases/module_with_default_argument.py new file mode 100644 index 000000000000..d6a42e8ba29c --- /dev/null +++ b/sdks/python/apache_beam/internal/test_cases/module_with_default_argument.py @@ -0,0 +1,5 @@ +"""Module for testing code path generation with functions.""" + + +def function_with_lambda_default_argument(fn=lambda x: 1): + return fn From f1ce3ca7056adddc6fbebfe2d405fd39dd1edf9b Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 13:42:30 -0400 Subject: [PATCH 018/142] Create pickle_code_path_test.py --- .../internal/pickle_code_path_test.py | 423 ++++++++++++++++++ 1 file changed, 423 insertions(+) create mode 100644 sdks/python/apache_beam/internal/pickle_code_path_test.py diff --git a/sdks/python/apache_beam/internal/pickle_code_path_test.py b/sdks/python/apache_beam/internal/pickle_code_path_test.py new file mode 100644 index 000000000000..3b27ff529539 --- /dev/null +++ b/sdks/python/apache_beam/internal/pickle_code_path_test.py @@ -0,0 +1,423 @@ +"""Tests for generating stable identifiers for Python __code__ to use for Pickle serialization.""" + +from apache_beam.internal import code_object_pickler +from apache_beam.internal.test_cases import after_module_add_function +from apache_beam.internal.test_cases import after_module_add_lambda_variable +from apache_beam.internal.test_cases import after_module_add_variable +from apache_beam.internal.test_cases import after_module_remove_lambda_variable +from apache_beam.internal.test_cases import after_module_remove_variable +from apache_beam.internal.test_cases import after_module_with_classes +from apache_beam.internal.test_cases import after_module_with_global_variable +from apache_beam.internal.test_cases import after_module_with_nested_function +from apache_beam.internal.test_cases import after_module_with_nested_function_2 +from apache_beam.internal.test_cases import after_module_with_single_class +from apache_beam.internal.test_cases import before_module_with_classes +from apache_beam.internal.test_cases import before_module_with_functions +from apache_beam.internal.test_cases import before_module_with_lambdas +from apache_beam.internal.test_cases import module_with_default_argument + +from google3.testing.pybase import googletest + + +class CodePathTest(googletest.TestCase): + + def test_get_code_from_stable_reference_empty_path(self): + with self.assertRaisesRegex(ValueError, "Path must not be empty"): + code_object_pickler._get_code_from_stable_reference("") + + def test_get_code_from_stable_reference_invalid_default_index(self): + with self.assertRaisesRegex(ValueError, "out of bounds"): + code_object_pickler._get_code_from_stable_reference( + "apache_beam.internal.test_cases.module_with_default_argument." + "function_with_lambda_default_argument.__defaults__[1]" + ) + + def test_get_code_from_stable_reference_invalid_single_name_path(self): + with self.assertRaisesRegex( + AttributeError, "Could not find code object with path" + ): + code_object_pickler._get_code_from_stable_reference( + "apache_beam.internal.test_cases.before_module_with_lambdas." + "my_function.__code__.co_consts[something]" + ) + + def test_get_code_from_stable_reference_invalid_lambda_with_args_path(self): + with self.assertRaisesRegex( + AttributeError, "Could not find code object with path" + ): + code_object_pickler._get_code_from_stable_reference( + "apache_beam.internal.test_cases.before_module_with_lambdas." + "my_function.__code__.co_consts[, ('x',)]" + ) + + def test_get_code_from_stable_reference_invalid_lambda_with_hash_path(self): + with self.assertRaisesRegex( + AttributeError, "Could not find code object with path" + ): + code_object_pickler._get_code_from_stable_reference( + "apache_beam.internal.test_cases.before_module_with_lambdas." + "my_function.__code__.co_consts[, ('',), 1234567890]" + ) + + def test_get_code_add_local_variable_in_class(self): + self.assertEqual( + code_object_pickler._get_code_from_stable_reference( + code_object_pickler._get_code_path( + before_module_with_classes.AddLocalVariable.my_method() + ).replace("before_module_with_classes", "after_module_with_classes") + ), + after_module_with_classes.AddLocalVariable.my_method().__code__, + ) + + def test_get_code_remove_local_variable_in_class(self): + self.assertEqual( + code_object_pickler._get_code_from_stable_reference( + code_object_pickler._get_code_path( + before_module_with_classes.RemoveLocalVariable.my_method() + ).replace("before_module_with_classes", "after_module_with_classes") + ), + after_module_with_classes.RemoveLocalVariable.my_method().__code__, + ) + + def test_get_code_add_lambda_variable_in_class(self): + self.assertEqual( + code_object_pickler._get_code_from_stable_reference( + code_object_pickler._get_code_path( + before_module_with_classes.AddLambdaVariable.my_method() + ).replace("before_module_with_classes", "after_module_with_classes") + ), + after_module_with_classes.AddLambdaVariable.my_method().__code__, + ) + + def test_get_code_remove_lambda_variable_in_class(self): + with self.assertRaisesRegex(AttributeError, "object has no attribute"): + code_object_pickler._get_code_from_stable_reference( + code_object_pickler._get_code_path( + before_module_with_classes.RemoveLambdaVariable.my_method() + ).replace("before_module_with_classes", "after_module_with_classes") + ) + + def test_get_code_nested_function_in_class(self): + self.assertEqual( + code_object_pickler._get_code_from_stable_reference( + code_object_pickler._get_code_path( + before_module_with_classes.ClassWithNestedFunction.my_method() + ).replace("before_module_with_classes", "after_module_with_classes") + ), + after_module_with_classes.ClassWithNestedFunction.my_method().__code__, + ) + + def test_get_code_nested_function_2_in_class(self): + self.assertEqual( + code_object_pickler._get_code_from_stable_reference( + code_object_pickler._get_code_path( + before_module_with_classes.ClassWithNestedFunction2.my_method() + ).replace("before_module_with_classes", "after_module_with_classes") + ), + after_module_with_classes.ClassWithNestedFunction2.my_method().__code__, + ) + + def test_get_code_add_new_function_in_class(self): + self.assertEqual( + code_object_pickler._get_code_from_stable_reference( + code_object_pickler._get_code_path( + before_module_with_classes.ClassWithTwoMethods.my_method() + ).replace("before_module_with_classes", "after_module_with_classes") + ), + after_module_with_classes.ClassWithTwoMethods.my_method().__code__, + ) + + def test_get_code_remove_method_in_class(self): + self.assertEqual( + code_object_pickler._get_code_from_stable_reference( + code_object_pickler._get_code_path( + before_module_with_classes.RemoveMethod.my_method() + ).replace("before_module_with_classes", "after_module_with_classes") + ), + after_module_with_classes.RemoveMethod.my_method().__code__, + ) + + def test_get_code_add_global_variable(self): + self.assertEqual( + code_object_pickler._get_code_from_stable_reference( + code_object_pickler._get_code_path( + before_module_with_functions.my_function() + ).replace( + "before_module_with_functions", + "after_module_with_global_variable", + ) + ), + after_module_with_global_variable.my_function().__code__, + ) + + def test_get_code_add_top_level_function(self): + self.assertEqual( + code_object_pickler._get_code_from_stable_reference( + code_object_pickler._get_code_path( + before_module_with_functions.my_function() + ).replace( + "before_module_with_functions", "after_module_add_function" + ) + ), + after_module_add_function.my_function().__code__, + ) + + def test_get_code_add_local_variable_in_function(self): + self.assertEqual( + code_object_pickler._get_code_from_stable_reference( + code_object_pickler._get_code_path( + before_module_with_functions.my_function() + ).replace( + "before_module_with_functions", "after_module_add_variable" + ) + ), + after_module_add_variable.my_function().__code__, + ) + + def test_get_code_remove_local_variable_in_function(self): + self.assertEqual( + code_object_pickler._get_code_from_stable_reference( + code_object_pickler._get_code_path( + before_module_with_functions.my_function() + ).replace( + "before_module_with_functions", "after_module_remove_variable" + ) + ), + after_module_remove_variable.my_function().__code__, + ) + + def test_get_code_nested_function_in_function(self): + self.assertEqual( + code_object_pickler._get_code_from_stable_reference( + code_object_pickler._get_code_path( + before_module_with_functions.my_function() + ).replace( + "before_module_with_functions", + "after_module_with_nested_function", + ) + ), + after_module_with_nested_function.my_function().__code__, + ) + + def test_get_code_nested_function_2_in_function(self): + self.assertEqual( + code_object_pickler._get_code_from_stable_reference( + code_object_pickler._get_code_path( + before_module_with_functions.my_function() + ).replace( + "before_module_with_functions", + "after_module_with_nested_function_2", + ) + ), + after_module_with_nested_function_2.my_function().__code__, + ) + + def test_get_code_add_class(self): + self.assertEqual( + code_object_pickler._get_code_from_stable_reference( + code_object_pickler._get_code_path( + before_module_with_functions.my_function() + ).replace( + "before_module_with_functions", "after_module_with_single_class" + ) + ), + after_module_with_single_class.my_function().__code__, + ) + + def test_get_code_add_lambda_variable_in_function(self): + self.assertEqual( + code_object_pickler._get_code_from_stable_reference( + code_object_pickler._get_code_path( + before_module_with_functions.my_function() + ).replace( + "before_module_with_functions", + "after_module_add_lambda_variable", + ) + ), + after_module_add_lambda_variable.my_function().__code__, + ) + + def test_get_code_remove_lambda_variable_in_function(self): + with self.assertRaisesRegex(AttributeError, "object has no attribute"): + code_object_pickler._get_code_from_stable_reference( + code_object_pickler._get_code_path( + before_module_with_lambdas.my_function() + ).replace( + "before_module_with_lambdas", + "after_module_remove_lambda_variable", + ) + ) + + def test_identifiers_new_local_variable_in_class(self): + self.assertEqual( + code_object_pickler._get_code_path( + before_module_with_classes.AddLocalVariable.my_method() + ).replace("before_module_with_classes", "module_name"), + code_object_pickler._get_code_path( + after_module_with_classes.AddLocalVariable.my_method() + ).replace("after_module_with_classes", "module_name"), + ) + + def test_identifiers_remove_local_variable_in_class(self): + self.assertEqual( + code_object_pickler._get_code_path( + before_module_with_classes.RemoveLocalVariable.my_method() + ).replace("before_module_with_classes", "module_name"), + code_object_pickler._get_code_path( + after_module_with_classes.RemoveLocalVariable.my_method() + ).replace("after_module_with_classes", "module_name"), + ) + + def test_identifiers_add_lambda_variable_in_class(self): + self.assertNotEqual( + code_object_pickler._get_code_path( + before_module_with_classes.AddLambdaVariable.my_method() + ).replace("before_module_with_classes", "module_name"), + code_object_pickler._get_code_path( + after_module_with_classes.AddLambdaVariable.my_method() + ).replace("after_module_with_classes", "module_name"), + ) + + def test_identifiers_remove_lambda_variable_in_class(self): + self.assertNotEqual( + code_object_pickler._get_code_path( + before_module_with_classes.RemoveLambdaVariable.my_method() + ).replace("before_module_with_classes", "module_name"), + code_object_pickler._get_code_path( + after_module_with_classes.RemoveLambdaVariable.my_method() + ).replace("after_module_with_classes", "module_name"), + ) + + def test_identifiers_nested_function_in_class(self): + self.assertEqual( + code_object_pickler._get_code_path( + before_module_with_classes.ClassWithNestedFunction.my_method() + ).replace("before_module_with_classes", "module_name"), + code_object_pickler._get_code_path( + after_module_with_classes.ClassWithNestedFunction.my_method() + ).replace("after_module_with_classes", "module_name"), + ) + + def test_identifiers_nested_function_2_in_class(self): + self.assertEqual( + code_object_pickler._get_code_path( + before_module_with_classes.ClassWithNestedFunction2.my_method() + ).replace("before_module_with_classes", "module_name"), + code_object_pickler._get_code_path( + after_module_with_classes.ClassWithNestedFunction2.my_method() + ).replace("after_module_with_classes", "module_name"), + ) + + def test_identifiers_add_new_function_in_class(self): + self.assertEqual( + code_object_pickler._get_code_path( + before_module_with_classes.ClassWithTwoMethods.my_method() + ).replace("before_module_with_classes", "module_name"), + code_object_pickler._get_code_path( + after_module_with_classes.ClassWithTwoMethods.my_method() + ).replace("after_module_with_classes", "module_name"), + ) + + def test_identifiers_remove_function_in_class(self): + self.assertEqual( + code_object_pickler._get_code_path( + before_module_with_classes.ClassWithTwoMethods.my_method() + ).replace("before_module_with_classes", "module_name"), + code_object_pickler._get_code_path( + after_module_with_classes.ClassWithTwoMethods.my_method() + ).replace("after_module_with_classes", "module_name"), + ) + + def test_identifiers_add_global_variable(self): + self.assertEqual( + code_object_pickler._get_code_path( + before_module_with_functions.my_function() + ).replace("before_module_with_functions", "module_name"), + code_object_pickler._get_code_path( + after_module_with_global_variable.my_function() + ).replace("after_module_with_global_variable", "module_name"), + ) + + def test_identifiers_add_top_level_function(self): + self.assertEqual( + code_object_pickler._get_code_path( + before_module_with_functions.my_function() + ).replace("before_module_with_functions", "module_name"), + code_object_pickler._get_code_path( + after_module_add_function.my_function() + ).replace("after_module_add_function", "module_name"), + ) + + def test_identifiers_add_local_variable_in_function(self): + self.assertEqual( + code_object_pickler._get_code_path( + before_module_with_functions.my_function() + ).replace("before_module_with_functions", "module_name"), + code_object_pickler._get_code_path( + after_module_add_variable.my_function() + ).replace("after_module_add_variable", "module_name"), + ) + + def test_identifiers_remove_local_variable_in_function(self): + self.assertEqual( + code_object_pickler._get_code_path( + before_module_with_functions.my_function() + ).replace("before_module_with_functions", "module_name"), + code_object_pickler._get_code_path( + after_module_remove_variable.my_function() + ).replace("after_module_remove_variable", "module_name"), + ) + + def test_identifiers_nested_function_in_function(self): + self.assertEqual( + code_object_pickler._get_code_path( + before_module_with_functions.my_function() + ).replace("before_module_with_functions", "module_name"), + code_object_pickler._get_code_path( + after_module_with_nested_function.my_function() + ).replace("after_module_with_nested_function", "module_name"), + ) + + def test_identifiers_nested_function_2_in_function(self): + self.assertEqual( + code_object_pickler._get_code_path( + before_module_with_functions.my_function() + ).replace("before_module_with_functions", "module_name"), + code_object_pickler._get_code_path( + after_module_with_nested_function_2.my_function() + ).replace("after_module_with_nested_function_2", "module_name"), + ) + + def test_identifiers_add_class(self): + self.assertEqual( + code_object_pickler._get_code_path( + before_module_with_functions.my_function() + ).replace("before_module_with_functions", "module_name"), + code_object_pickler._get_code_path( + after_module_with_single_class.my_function() + ).replace("after_module_with_single_class", "module_name"), + ) + + def test_identifiers_add_lambda_variable_in_function(self): + self.assertNotEqual( + code_object_pickler._get_code_path( + before_module_with_functions.my_function() + ).replace("before_module_with_functions", "module_name"), + code_object_pickler._get_code_path( + after_module_add_lambda_variable.my_function() + ).replace("after_module_add_lambda_variable", "module_name"), + ) + + def test_identifiers_remove_lambda_variable_in_function(self): + self.assertNotEqual( + code_object_pickler._get_code_path( + before_module_with_lambdas.my_function() + ).replace("before_module_with_lambdas", "module_name"), + code_object_pickler._get_code_path( + after_module_remove_lambda_variable.my_function() + ).replace("after_module_remove_lambda_variable", "module_name"), + ) + + +if __name__ == "__main__": + googletest.main() From 2e9ff8bf69df3d877d9867d9ad68e6310063bbe2 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 13:46:21 -0400 Subject: [PATCH 019/142] Update code_object_pickler.py --- sdks/python/apache_beam/internal/code_object_pickler.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sdks/python/apache_beam/internal/code_object_pickler.py b/sdks/python/apache_beam/internal/code_object_pickler.py index e38a3b740156..500fc2628879 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler.py +++ b/sdks/python/apache_beam/internal/code_object_pickler.py @@ -15,6 +15,13 @@ # limitations under the License. # +import collections +import hashlib +import inspect +import os +import re +import sys +import types def get_normalized_path(path): """Returns a normalized path. This function is intended to be overridden.""" From d940cfa65c92d334dd9f4ceec7e98fcc15c6cb4d Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 13:49:23 -0400 Subject: [PATCH 020/142] Update pickle_code_path_test.py --- sdks/python/apache_beam/internal/pickle_code_path_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdks/python/apache_beam/internal/pickle_code_path_test.py b/sdks/python/apache_beam/internal/pickle_code_path_test.py index 3b27ff529539..388418022b68 100644 --- a/sdks/python/apache_beam/internal/pickle_code_path_test.py +++ b/sdks/python/apache_beam/internal/pickle_code_path_test.py @@ -1,5 +1,7 @@ """Tests for generating stable identifiers for Python __code__ to use for Pickle serialization.""" +import unittest + from apache_beam.internal import code_object_pickler from apache_beam.internal.test_cases import after_module_add_function from apache_beam.internal.test_cases import after_module_add_lambda_variable @@ -16,10 +18,8 @@ from apache_beam.internal.test_cases import before_module_with_lambdas from apache_beam.internal.test_cases import module_with_default_argument -from google3.testing.pybase import googletest - -class CodePathTest(googletest.TestCase): +class CodePathTest(unittest.TestCase): def test_get_code_from_stable_reference_empty_path(self): with self.assertRaisesRegex(ValueError, "Path must not be empty"): @@ -420,4 +420,4 @@ def test_identifiers_remove_lambda_variable_in_function(self): if __name__ == "__main__": - googletest.main() + unittest.main() From 0438ba447f8f83610653b50aeb712ad5bda45eb4 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 14:06:55 -0400 Subject: [PATCH 021/142] Update code_object_pickler_test.py --- .../internal/code_object_pickler_test.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index be5173a6f09f..bc22f91e5cba 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + import unittest from apache_beam.internal import code_object_pickler From 30ece4b636a8b95f840814187e1f5809bebe24dd Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 14:07:22 -0400 Subject: [PATCH 022/142] Update pickle_code_path_test.py --- .../internal/pickle_code_path_test.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sdks/python/apache_beam/internal/pickle_code_path_test.py b/sdks/python/apache_beam/internal/pickle_code_path_test.py index 388418022b68..65f7ec54bd00 100644 --- a/sdks/python/apache_beam/internal/pickle_code_path_test.py +++ b/sdks/python/apache_beam/internal/pickle_code_path_test.py @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + """Tests for generating stable identifiers for Python __code__ to use for Pickle serialization.""" import unittest From 3dc68567579c88f303ed617e183d8b06bfec0af5 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 14:07:37 -0400 Subject: [PATCH 023/142] Update after_module_add_function.py --- .../test_cases/after_module_add_function.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py b/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py index 06ed3283b78f..6c9d83567da5 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + """Module for testing code path generation with functions.""" From 74cb523e14457f10d3a13b627b33ecbec692c5be Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 14:07:46 -0400 Subject: [PATCH 024/142] Update after_module_add_lambda_variable.py --- .../after_module_add_lambda_variable.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_add_lambda_variable.py b/sdks/python/apache_beam/internal/test_cases/after_module_add_lambda_variable.py index 205fa64ceaed..5b2650692618 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_add_lambda_variable.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_add_lambda_variable.py @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + """Module for testing code path generation with functions.""" def my_function(): From 82e5656a6a79cdd46203db9d385cef25d4d80c82 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 14:07:59 -0400 Subject: [PATCH 025/142] Update after_module_add_variable.py --- .../test_cases/after_module_add_variable.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_add_variable.py b/sdks/python/apache_beam/internal/test_cases/after_module_add_variable.py index 788b988c580a..bf70147a9027 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_add_variable.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_add_variable.py @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + """Module for testing code path generation with functions.""" From 0c0cf8720bbe3b2d4086067f3b796919db4c79e6 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 14:08:08 -0400 Subject: [PATCH 026/142] Update after_module_remove_lambda_variable.py --- .../after_module_remove_lambda_variable.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_remove_lambda_variable.py b/sdks/python/apache_beam/internal/test_cases/after_module_remove_lambda_variable.py index b24f5bfc2643..436f42877cf7 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_remove_lambda_variable.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_remove_lambda_variable.py @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + """Module for testing code path generation with functions.""" From 50011e4bd48caeef9d40872a1d4ecb4dccacf083 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 14:09:31 -0400 Subject: [PATCH 027/142] Update after_module_remove_variable.py --- .../test_cases/after_module_remove_variable.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_remove_variable.py b/sdks/python/apache_beam/internal/test_cases/after_module_remove_variable.py index b24f5bfc2643..436f42877cf7 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_remove_variable.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_remove_variable.py @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + """Module for testing code path generation with functions.""" From 35f2dfe13e993a0179efd7164973c68e8a7e18d4 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 14:09:42 -0400 Subject: [PATCH 028/142] Update after_module_with_classes.py --- .../test_cases/after_module_with_classes.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py index 20f4e6a59a64..1546c1c6e408 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + """Module for testing code path generation with classes.""" From 256a010b6425b883bc4dca8ee989925978e87a3a Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 14:09:54 -0400 Subject: [PATCH 029/142] Update after_module_with_global_variable.py --- .../after_module_with_global_variable.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_global_variable.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_global_variable.py index d54f76626291..670e4108f737 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_with_global_variable.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_global_variable.py @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + """Module for testing code path generation with a global variable.""" GLOBAL_VARIABLE = lambda: 3 From 4764deaa2584e34a09be93a02a82e2be5415fac9 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 14:10:03 -0400 Subject: [PATCH 030/142] Update after_module_with_nested_function.py --- .../after_module_with_nested_function.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py index 1f1c3ad9a9ab..53b2e6a012f8 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + """Module for testing code path generation with functions.""" From a65db8b1afe737a6a20d80c8df1caf886643cc0a Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 14:10:13 -0400 Subject: [PATCH 031/142] Update after_module_with_nested_function_2.py --- .../after_module_with_nested_function_2.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function_2.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function_2.py index 8de902e559d0..a14a0e1b6021 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function_2.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function_2.py @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + """Module for testing code path generation with functions.""" From dca267fa256b172c58bd46b56ad8e26829f0bee0 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 14:10:23 -0400 Subject: [PATCH 032/142] Update after_module_with_single_class.py --- .../after_module_with_single_class.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_single_class.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_single_class.py index 79151f16373e..dab476321fa2 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_with_single_class.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_single_class.py @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + """Module for testing code path generation with a single class.""" From 4b21993f5ea839b68aacea27e017ed6df5c515e0 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 14:10:33 -0400 Subject: [PATCH 033/142] Update before_module_with_classes.py --- .../test_cases/before_module_with_classes.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sdks/python/apache_beam/internal/test_cases/before_module_with_classes.py b/sdks/python/apache_beam/internal/test_cases/before_module_with_classes.py index 4f3b8fb2e63c..cf9c60bde844 100644 --- a/sdks/python/apache_beam/internal/test_cases/before_module_with_classes.py +++ b/sdks/python/apache_beam/internal/test_cases/before_module_with_classes.py @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + """Module for testing code path generation with classes.""" From 996057da8d2b78ab157bf1fac42a398b1a3a06be Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 14:10:42 -0400 Subject: [PATCH 034/142] Update before_module_with_functions.py --- .../test_cases/before_module_with_functions.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sdks/python/apache_beam/internal/test_cases/before_module_with_functions.py b/sdks/python/apache_beam/internal/test_cases/before_module_with_functions.py index 34614ca51d7c..67e2a7be28f4 100644 --- a/sdks/python/apache_beam/internal/test_cases/before_module_with_functions.py +++ b/sdks/python/apache_beam/internal/test_cases/before_module_with_functions.py @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + """Module for testing code path generation with functions.""" From 4ac38648aed5802da833314cef5a32d52d2eac09 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 14:10:52 -0400 Subject: [PATCH 035/142] Update before_module_with_lambdas.py --- .../test_cases/before_module_with_lambdas.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sdks/python/apache_beam/internal/test_cases/before_module_with_lambdas.py b/sdks/python/apache_beam/internal/test_cases/before_module_with_lambdas.py index 5265c4ce3c71..88b570f18dc0 100644 --- a/sdks/python/apache_beam/internal/test_cases/before_module_with_lambdas.py +++ b/sdks/python/apache_beam/internal/test_cases/before_module_with_lambdas.py @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + """Module for testing code path generation with functions.""" From cc91dfec2bc9d5747153aa883eb45de9dd3324a7 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 14:11:04 -0400 Subject: [PATCH 036/142] Update module_with_default_argument.py --- .../test_cases/module_with_default_argument.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/sdks/python/apache_beam/internal/test_cases/module_with_default_argument.py b/sdks/python/apache_beam/internal/test_cases/module_with_default_argument.py index d6a42e8ba29c..ae3366ae752a 100644 --- a/sdks/python/apache_beam/internal/test_cases/module_with_default_argument.py +++ b/sdks/python/apache_beam/internal/test_cases/module_with_default_argument.py @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + """Module for testing code path generation with functions.""" From 69803874e49e9d08d5340e71e36c47964f867c39 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 14:31:18 -0400 Subject: [PATCH 037/142] Update after_module_add_lambda_variable.py --- .../internal/test_cases/after_module_add_lambda_variable.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_add_lambda_variable.py b/sdks/python/apache_beam/internal/test_cases/after_module_add_lambda_variable.py index 5b2650692618..1b7366bfda0f 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_add_lambda_variable.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_add_lambda_variable.py @@ -17,6 +17,7 @@ """Module for testing code path generation with functions.""" + def my_function(): a = 1 b = lambda: 2 From 46de2064f4265a878e5207f7f9a71a24a560a62d Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 14:32:35 -0400 Subject: [PATCH 038/142] Update after_module_with_nested_function.py --- .../internal/test_cases/after_module_with_nested_function.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py index 53b2e6a012f8..64ec6c0c6fe6 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py @@ -25,4 +25,3 @@ def nested_function(): c = 3 return c return b - From 031386a4f4b45bc9143d82f9d3a6a2584adbc7c5 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 14:34:35 -0400 Subject: [PATCH 039/142] Update code_object_pickler.py --- sdks/python/apache_beam/internal/code_object_pickler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdks/python/apache_beam/internal/code_object_pickler.py b/sdks/python/apache_beam/internal/code_object_pickler.py index 500fc2628879..96ac8479421f 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler.py +++ b/sdks/python/apache_beam/internal/code_object_pickler.py @@ -23,10 +23,12 @@ import sys import types + def get_normalized_path(path): """Returns a normalized path. This function is intended to be overridden.""" return path + def _get_code_path(callable): """Returns the stable reference to the code object. From 640f90b2253891fc633155c3b445d0fbee9ea30e Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 14:56:37 -0400 Subject: [PATCH 040/142] Update code_object_pickler_test.py --- sdks/python/apache_beam/internal/code_object_pickler_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index bc22f91e5cba..171282988b08 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -18,7 +18,7 @@ import unittest from apache_beam.internal import code_object_pickler -from google3.testing.pybase import parameterized +from parameterized import parameterized def top_level_function(): From ef7c54efff37f6921fee533ec846f43295aa241c Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 16:01:35 -0400 Subject: [PATCH 041/142] Update code_object_pickler_test.py --- sdks/python/apache_beam/internal/code_object_pickler_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index 171282988b08..d1d8dd9ab042 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -18,7 +18,7 @@ import unittest from apache_beam.internal import code_object_pickler -from parameterized import parameterized +from absl.testing import parameterized def top_level_function(): From 6907af4f20beabe53ea387f27e1f509032dac3a4 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 16:28:52 -0400 Subject: [PATCH 042/142] Update code_object_pickler_test.py --- .../internal/code_object_pickler_test.py | 112 ++++++++++-------- 1 file changed, 61 insertions(+), 51 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index d1d8dd9ab042..f2921610391f 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -31,22 +31,17 @@ def top_level_function(): def get_nested_function(): def nested_function(): return 1 + return nested_function def get_lambda_from_dictionary(): - d = { - "a": lambda x: 1, - "b": lambda y: 2 - } + d = {"a": lambda x: 1, "b": lambda y: 2} return d["a"] def get_lambda_from_dictionary_same_args(): - d = { - "a": lambda x: 1, - "b": lambda x: x + 1 - } + d = {"a": lambda x: 1, "b": lambda x: x + 1} return d["a"] @@ -88,6 +83,7 @@ class ClassWithNestedFunction: def process(self): def nested_function(): return 1 + return nested_function @@ -105,56 +101,69 @@ def process(self): class ClassWithNestedLambda: def process(self): def get_lambda_from_dictionary(): - d = { - "a": lambda x: 1, - "b": lambda y: 2 - } + d = {"a": lambda x: 1, "b": lambda y: 2} return d["a"] + return get_lambda_from_dictionary() + test_cases = [ - (top_level_function, "__main__.top_level_function.__code__"), - (top_level_lambda, "__main__.top_level_lambda.__code__"), - (get_nested_function(), - ("__main__.get_nested_function.__code__.co_consts" - "[nested_function]")), - (get_lambda_from_dictionary(), - ("__main__.get_lambda_from_dictionary.__code__.co_consts" - "[, ('x',)]")), - (get_lambda_from_dictionary_same_args(), - ("__main__.get_lambda_from_dictionary_same_args.__code__.co_consts" - "[, ('x',), " - "ac455edc80140a7f12ad229519456f71]")), - (function_with_lambda_default_argument(), - ("__main__.function_with_lambda_default_argument.__defaults__[0]" - ".__code__")), - (function_with_function_default_argument(), - "__main__.top_level_function.__code__"), - (add_one, - "__main__.function_decorator.__code__.co_consts[]"), - (ClassWithFunction.process, - "__main__.ClassWithFunction.process.__code__"), - (ClassWithStaticMethod.static_method, - "__main__.ClassWithStaticMethod.static_method.__code__"), - (ClassWithClassMethod.class_method, - "__main__.ClassWithClassMethod.class_method.__code__"), - (ClassWithNestedFunction().process(), - ("__main__.ClassWithNestedFunction.process.__code__.co_consts" - "[nested_function]")), - (ClassWithLambda().process(), - "__main__.ClassWithLambda.process.__code__.co_consts[]"), - (ClassWithNestedClass.InnerClass().process, - "__main__.ClassWithNestedClass.InnerClass.process.__code__"), - (ClassWithNestedLambda().process(), - ("__main__.ClassWithNestedLambda.process.__code__.co_consts" - "[get_lambda_from_dictionary].co_consts[, ('x',)]")), - (ClassWithNestedLambda.process, - "__main__.ClassWithNestedLambda.process.__code__"), - ] + (top_level_function, "__main__.top_level_function.__code__"), + (top_level_lambda, "__main__.top_level_lambda.__code__"), + ( + get_nested_function(), + ("__main__.get_nested_function.__code__.co_consts" + "[nested_function]")), + ( + get_lambda_from_dictionary(), + ( + "__main__.get_lambda_from_dictionary.__code__.co_consts" + "[, ('x',)]")), + ( + get_lambda_from_dictionary_same_args(), + ( + "__main__.get_lambda_from_dictionary_same_args.__code__.co_consts" + "[, ('x',), " + "ac455edc80140a7f12ad229519456f71]")), + ( + function_with_lambda_default_argument(), + ( + "__main__.function_with_lambda_default_argument.__defaults__[0]" + ".__code__")), + ( + function_with_function_default_argument(), + "__main__.top_level_function.__code__"), + (add_one, "__main__.function_decorator.__code__.co_consts[]"), + (ClassWithFunction.process, "__main__.ClassWithFunction.process.__code__"), + ( + ClassWithStaticMethod.static_method, + "__main__.ClassWithStaticMethod.static_method.__code__"), + ( + ClassWithClassMethod.class_method, + "__main__.ClassWithClassMethod.class_method.__code__"), + ( + ClassWithNestedFunction().process(), + ( + "__main__.ClassWithNestedFunction.process.__code__.co_consts" + "[nested_function]")), + ( + ClassWithLambda().process(), + "__main__.ClassWithLambda.process.__code__.co_consts[]"), + ( + ClassWithNestedClass.InnerClass().process, + "__main__.ClassWithNestedClass.InnerClass.process.__code__"), + ( + ClassWithNestedLambda().process(), + ( + "__main__.ClassWithNestedLambda.process.__code__.co_consts" + "[get_lambda_from_dictionary].co_consts[, ('x',)]")), + ( + ClassWithNestedLambda.process, + "__main__.ClassWithNestedLambda.process.__code__"), +] class DillTest(parameterized.TestCase): - @parameterized.parameters(test_cases) def test_get_code_path(self, callable, expected): actual = code_object_pickler._get_code_path(callable) @@ -171,6 +180,7 @@ def test_roundtrip(self, callable, _): actual = code_object_pickler._get_code_from_stable_reference(path) self.assertEqual(actual, callable.__code__) + if __name__ == "__main__": unittest.main() From 11bc736f88b4322c2475d18fbf00c83e9fb7956b Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 16:33:50 -0400 Subject: [PATCH 043/142] Update code_object_pickler.py --- .../internal/code_object_pickler.py | 43 +++++++------------ 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler.py b/sdks/python/apache_beam/internal/code_object_pickler.py index 96ac8479421f..3e0d71377ec7 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler.py +++ b/sdks/python/apache_beam/internal/code_object_pickler.py @@ -116,11 +116,11 @@ def _search_module_or_class(callable, node, qual_name_parts): if first_part == '': for name in dir(node): value = getattr(node, name) - if (isinstance(value, type(callable)) - and value.__code__ == callable.__code__): + if (isinstance(value, type(callable)) and + value.__code__ == callable.__code__): return name + '.__code__' - elif (isinstance(value, types.FunctionType) - and value.__defaults__ is not None): + elif (isinstance(value, types.FunctionType) and + value.__defaults__ is not None): # Python functions can have other functions as default parameters which # might contain the code object so we have to search them. for i, default_param_value in enumerate(value.__defaults__): @@ -129,8 +129,7 @@ def _search_module_or_class(callable, node, qual_name_parts): return _extend_path(name, _extend_path(f'__defaults__[{i}]', path)) else: return _extend_path( - first_part, _search(callable, getattr(node, first_part), rest) - ) + first_part, _search(callable, getattr(node, first_part), rest)) def _search_function(callable, node, qual_name_parts): @@ -154,8 +153,7 @@ def _search_function(callable, node, qual_name_parts): # the code object of the function. if first_part == '': return _extend_path( - '__code__', _search(callable, node.__code__, qual_name_parts) - ) + '__code__', _search(callable, node.__code__, qual_name_parts)) def _search_code(callable, node, qual_name_parts): @@ -247,12 +245,10 @@ def _search_lambda(callable, code_objects_by_name, qual_name_parts): _SINGLE_NAME_PATTERN = re.compile(r'co_consts\[([a-zA-Z0-9\<\>_-]+)]') # Matches a path like: co_consts[, ('x',)] _LAMBDA_WITH_ARGS_PATTERN = re.compile( - r"co_consts\[(<[^>]+>),\s*(\('[^']*'\s*,\s*\))\]" -) + r"co_consts\[(<[^>]+>),\s*(\('[^']*'\s*,\s*\))\]") # Matches a path like: co_consts[, ('x',), 1234567890] _LAMBDA_WITH_HASH_PATTERN = re.compile( - r"co_consts\[(<[^>]+>),\s*(\('[^']*'\s*,\s*\)),\s*(.+)\]" -) + r"co_consts\[(<[^>]+>),\s*(\('[^']*'\s*,\s*\)),\s*(.+)\]") # Matches a path like: __defaults__[0] _DEFAULT_PATTERN = re.compile(r'(__defaults__)\[(\d+)\]') # Matches an argument like: 'x' @@ -284,8 +280,7 @@ def _get_code_object_from_single_name_pattern(obj, name_result, path): def _get_code_object_from_lambda_with_args_pattern( - obj, lambda_with_args_result, path -): + obj, lambda_with_args_result, path): """Returns the code object from a lambda with args pattern. Args: @@ -304,8 +299,7 @@ def _get_code_object_from_lambda_with_args_pattern( for name, objects in code_objects.items(): for obj_ in objects: args = tuple( - re.findall(_ARGUMENT_PATTERN, lambda_with_args_result.group(2)) - ) + re.findall(_ARGUMENT_PATTERN, lambda_with_args_result.group(2))) if obj_.co_varnames == args: return obj_ raise AttributeError(f'Could not find code object with path: {path}') @@ -332,8 +326,7 @@ def _get_code_object_from_lambda_with_hash_pattern( for name, objects in code_objects.items(): for obj_ in objects: args = tuple( - re.findall(_ARGUMENT_PATTERN, lambda_with_hash_result.group(2)) - ) + re.findall(_ARGUMENT_PATTERN, lambda_with_hash_result.group(2))) if obj_.co_varnames == args: hash_value = lambda_with_hash_result.group(3) if hash_value == str(_create_bytecode_hash(obj_)): @@ -363,19 +356,16 @@ def _get_code_from_stable_reference(path): obj = _get_code_object_from_single_name_pattern(obj, name_result, path) elif lambda_with_args_result := _LAMBDA_WITH_ARGS_PATTERN.fullmatch(part): obj = _get_code_object_from_lambda_with_args_pattern( - obj, lambda_with_args_result, path - ) + obj, lambda_with_args_result, path) elif lambda_with_hash_result := _LAMBDA_WITH_HASH_PATTERN.fullmatch(part): obj = _get_code_object_from_lambda_with_hash_pattern( - obj, lambda_with_hash_result, path - ) + obj, lambda_with_hash_result, path) elif default_result := _DEFAULT_PATTERN.fullmatch(part): index = int(default_result.group(2)) if index >= len(obj.__defaults__): raise ValueError( f'Index {index} is out of bounds for obj.__defaults__' - f' {len(obj.__defaults__)} in path {path}' - ) + f' {len(obj.__defaults__)} in path {path}') obj = getattr(obj, '__defaults__')[index] else: obj = getattr(obj, part) @@ -396,9 +386,8 @@ def _signature(obj): """ if isinstance(obj, types.CodeType): arg_count = ( - obj.co_argcount - + obj.co_kwonlyargcount - + (obj.co_flags & 4 == 4) # PyCF_VARARGS + obj.co_argcount + obj.co_kwonlyargcount + + (obj.co_flags & 4 == 4) # PyCF_VARARGS + (obj.co_flags & 8 == 8) # PyCF_VARKEYWORDS ) return obj.co_varnames[:arg_count] From 23d2aea2e30b5ac3919be360784d814871223b06 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 16:51:39 -0400 Subject: [PATCH 044/142] Update pickle_code_path_test.py --- .../internal/pickle_code_path_test.py | 275 ++++++++---------- 1 file changed, 119 insertions(+), 156 deletions(-) diff --git a/sdks/python/apache_beam/internal/pickle_code_path_test.py b/sdks/python/apache_beam/internal/pickle_code_path_test.py index 65f7ec54bd00..8029fee708a8 100644 --- a/sdks/python/apache_beam/internal/pickle_code_path_test.py +++ b/sdks/python/apache_beam/internal/pickle_code_path_test.py @@ -37,7 +37,6 @@ class CodePathTest(unittest.TestCase): - def test_get_code_from_stable_reference_empty_path(self): with self.assertRaisesRegex(ValueError, "Path must not be empty"): code_object_pickler._get_code_from_stable_reference("") @@ -46,43 +45,35 @@ def test_get_code_from_stable_reference_invalid_default_index(self): with self.assertRaisesRegex(ValueError, "out of bounds"): code_object_pickler._get_code_from_stable_reference( "apache_beam.internal.test_cases.module_with_default_argument." - "function_with_lambda_default_argument.__defaults__[1]" - ) + "function_with_lambda_default_argument.__defaults__[1]") def test_get_code_from_stable_reference_invalid_single_name_path(self): with self.assertRaisesRegex( - AttributeError, "Could not find code object with path" - ): + AttributeError, "Could not find code object with path"): code_object_pickler._get_code_from_stable_reference( "apache_beam.internal.test_cases.before_module_with_lambdas." - "my_function.__code__.co_consts[something]" - ) + "my_function.__code__.co_consts[something]") def test_get_code_from_stable_reference_invalid_lambda_with_args_path(self): with self.assertRaisesRegex( - AttributeError, "Could not find code object with path" - ): + AttributeError, "Could not find code object with path"): code_object_pickler._get_code_from_stable_reference( "apache_beam.internal.test_cases.before_module_with_lambdas." - "my_function.__code__.co_consts[, ('x',)]" - ) + "my_function.__code__.co_consts[, ('x',)]") def test_get_code_from_stable_reference_invalid_lambda_with_hash_path(self): with self.assertRaisesRegex( - AttributeError, "Could not find code object with path" - ): + AttributeError, "Could not find code object with path"): code_object_pickler._get_code_from_stable_reference( "apache_beam.internal.test_cases.before_module_with_lambdas." - "my_function.__code__.co_consts[, ('',), 1234567890]" - ) + "my_function.__code__.co_consts[, ('',), 1234567890]") def test_get_code_add_local_variable_in_class(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_classes.AddLocalVariable.my_method() - ).replace("before_module_with_classes", "after_module_with_classes") - ), + before_module_with_classes.AddLocalVariable.my_method()). + replace("before_module_with_classes", "after_module_with_classes")), after_module_with_classes.AddLocalVariable.my_method().__code__, ) @@ -90,9 +81,8 @@ def test_get_code_remove_local_variable_in_class(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_classes.RemoveLocalVariable.my_method() - ).replace("before_module_with_classes", "after_module_with_classes") - ), + before_module_with_classes.RemoveLocalVariable.my_method()). + replace("before_module_with_classes", "after_module_with_classes")), after_module_with_classes.RemoveLocalVariable.my_method().__code__, ) @@ -100,9 +90,8 @@ def test_get_code_add_lambda_variable_in_class(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_classes.AddLambdaVariable.my_method() - ).replace("before_module_with_classes", "after_module_with_classes") - ), + before_module_with_classes.AddLambdaVariable.my_method()). + replace("before_module_with_classes", "after_module_with_classes")), after_module_with_classes.AddLambdaVariable.my_method().__code__, ) @@ -110,17 +99,16 @@ def test_get_code_remove_lambda_variable_in_class(self): with self.assertRaisesRegex(AttributeError, "object has no attribute"): code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_classes.RemoveLambdaVariable.my_method() - ).replace("before_module_with_classes", "after_module_with_classes") + before_module_with_classes.RemoveLambdaVariable.my_method()). + replace("before_module_with_classes", "after_module_with_classes") ) def test_get_code_nested_function_in_class(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_classes.ClassWithNestedFunction.my_method() - ).replace("before_module_with_classes", "after_module_with_classes") - ), + before_module_with_classes.ClassWithNestedFunction.my_method()). + replace("before_module_with_classes", "after_module_with_classes")), after_module_with_classes.ClassWithNestedFunction.my_method().__code__, ) @@ -128,9 +116,8 @@ def test_get_code_nested_function_2_in_class(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_classes.ClassWithNestedFunction2.my_method() - ).replace("before_module_with_classes", "after_module_with_classes") - ), + before_module_with_classes.ClassWithNestedFunction2.my_method()). + replace("before_module_with_classes", "after_module_with_classes")), after_module_with_classes.ClassWithNestedFunction2.my_method().__code__, ) @@ -138,9 +125,8 @@ def test_get_code_add_new_function_in_class(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_classes.ClassWithTwoMethods.my_method() - ).replace("before_module_with_classes", "after_module_with_classes") - ), + before_module_with_classes.ClassWithTwoMethods.my_method()). + replace("before_module_with_classes", "after_module_with_classes")), after_module_with_classes.ClassWithTwoMethods.my_method().__code__, ) @@ -148,9 +134,8 @@ def test_get_code_remove_method_in_class(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_classes.RemoveMethod.my_method() - ).replace("before_module_with_classes", "after_module_with_classes") - ), + before_module_with_classes.RemoveMethod.my_method()).replace( + "before_module_with_classes", "after_module_with_classes")), after_module_with_classes.RemoveMethod.my_method().__code__, ) @@ -158,12 +143,10 @@ def test_get_code_add_global_variable(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_functions.my_function() - ).replace( - "before_module_with_functions", - "after_module_with_global_variable", - ) - ), + before_module_with_functions.my_function()).replace( + "before_module_with_functions", + "after_module_with_global_variable", + )), after_module_with_global_variable.my_function().__code__, ) @@ -171,11 +154,9 @@ def test_get_code_add_top_level_function(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_functions.my_function() - ).replace( - "before_module_with_functions", "after_module_add_function" - ) - ), + before_module_with_functions.my_function()).replace( + "before_module_with_functions", + "after_module_add_function")), after_module_add_function.my_function().__code__, ) @@ -183,11 +164,9 @@ def test_get_code_add_local_variable_in_function(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_functions.my_function() - ).replace( - "before_module_with_functions", "after_module_add_variable" - ) - ), + before_module_with_functions.my_function()).replace( + "before_module_with_functions", + "after_module_add_variable")), after_module_add_variable.my_function().__code__, ) @@ -195,11 +174,9 @@ def test_get_code_remove_local_variable_in_function(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_functions.my_function() - ).replace( - "before_module_with_functions", "after_module_remove_variable" - ) - ), + before_module_with_functions.my_function()).replace( + "before_module_with_functions", + "after_module_remove_variable")), after_module_remove_variable.my_function().__code__, ) @@ -207,12 +184,9 @@ def test_get_code_nested_function_in_function(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_functions.my_function() - ).replace( - "before_module_with_functions", - "after_module_with_nested_function", - ) - ), + before_module_with_functions.my_function()).replace( + "before_module_with_functions", + "after_module_with_nested_function")), after_module_with_nested_function.my_function().__code__, ) @@ -220,12 +194,9 @@ def test_get_code_nested_function_2_in_function(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_functions.my_function() - ).replace( - "before_module_with_functions", - "after_module_with_nested_function_2", - ) - ), + before_module_with_functions.my_function()).replace( + "before_module_with_functions", + "after_module_with_nested_function_2")), after_module_with_nested_function_2.my_function().__code__, ) @@ -233,11 +204,9 @@ def test_get_code_add_class(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_functions.my_function() - ).replace( - "before_module_with_functions", "after_module_with_single_class" - ) - ), + before_module_with_functions.my_function()).replace( + "before_module_with_functions", + "after_module_with_single_class")), after_module_with_single_class.my_function().__code__, ) @@ -245,12 +214,9 @@ def test_get_code_add_lambda_variable_in_function(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_functions.my_function() - ).replace( - "before_module_with_functions", - "after_module_add_lambda_variable", - ) - ), + before_module_with_functions.my_function()).replace( + "before_module_with_functions", + "after_module_add_lambda_variable")), after_module_add_lambda_variable.my_function().__code__, ) @@ -258,181 +224,178 @@ def test_get_code_remove_lambda_variable_in_function(self): with self.assertRaisesRegex(AttributeError, "object has no attribute"): code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_lambdas.my_function() - ).replace( - "before_module_with_lambdas", - "after_module_remove_lambda_variable", - ) - ) + before_module_with_lambdas.my_function()).replace( + "before_module_with_lambdas", + "after_module_remove_lambda_variable")) def test_identifiers_new_local_variable_in_class(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_classes.AddLocalVariable.my_method() - ).replace("before_module_with_classes", "module_name"), + before_module_with_classes.AddLocalVariable.my_method()).replace( + "before_module_with_classes", "module_name"), code_object_pickler._get_code_path( - after_module_with_classes.AddLocalVariable.my_method() - ).replace("after_module_with_classes", "module_name"), + after_module_with_classes.AddLocalVariable.my_method()).replace( + "after_module_with_classes", "module_name"), ) def test_identifiers_remove_local_variable_in_class(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_classes.RemoveLocalVariable.my_method() - ).replace("before_module_with_classes", "module_name"), + before_module_with_classes.RemoveLocalVariable.my_method()).replace( + "before_module_with_classes", "module_name"), code_object_pickler._get_code_path( - after_module_with_classes.RemoveLocalVariable.my_method() - ).replace("after_module_with_classes", "module_name"), + after_module_with_classes.RemoveLocalVariable.my_method()).replace( + "after_module_with_classes", "module_name"), ) def test_identifiers_add_lambda_variable_in_class(self): self.assertNotEqual( code_object_pickler._get_code_path( - before_module_with_classes.AddLambdaVariable.my_method() - ).replace("before_module_with_classes", "module_name"), + before_module_with_classes.AddLambdaVariable.my_method()).replace( + "before_module_with_classes", "module_name"), code_object_pickler._get_code_path( - after_module_with_classes.AddLambdaVariable.my_method() - ).replace("after_module_with_classes", "module_name"), + after_module_with_classes.AddLambdaVariable.my_method()).replace( + "after_module_with_classes", "module_name"), ) def test_identifiers_remove_lambda_variable_in_class(self): self.assertNotEqual( code_object_pickler._get_code_path( - before_module_with_classes.RemoveLambdaVariable.my_method() - ).replace("before_module_with_classes", "module_name"), + before_module_with_classes.RemoveLambdaVariable.my_method()).replace( + "before_module_with_classes", "module_name"), code_object_pickler._get_code_path( - after_module_with_classes.RemoveLambdaVariable.my_method() - ).replace("after_module_with_classes", "module_name"), + after_module_with_classes.RemoveLambdaVariable.my_method()).replace( + "after_module_with_classes", "module_name"), ) def test_identifiers_nested_function_in_class(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_classes.ClassWithNestedFunction.my_method() - ).replace("before_module_with_classes", "module_name"), + before_module_with_classes.ClassWithNestedFunction.my_method()).replace( + "before_module_with_classes", "module_name"), code_object_pickler._get_code_path( - after_module_with_classes.ClassWithNestedFunction.my_method() - ).replace("after_module_with_classes", "module_name"), + after_module_with_classes.ClassWithNestedFunction.my_method()). + replace("after_module_with_classes", "module_name"), ) def test_identifiers_nested_function_2_in_class(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_classes.ClassWithNestedFunction2.my_method() - ).replace("before_module_with_classes", "module_name"), + before_module_with_classes.ClassWithNestedFunction2.my_method()).replace( + "before_module_with_classes", "module_name"), code_object_pickler._get_code_path( - after_module_with_classes.ClassWithNestedFunction2.my_method() - ).replace("after_module_with_classes", "module_name"), + after_module_with_classes.ClassWithNestedFunction2.my_method()). + replace("after_module_with_classes", "module_name"), ) def test_identifiers_add_new_function_in_class(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_classes.ClassWithTwoMethods.my_method() - ).replace("before_module_with_classes", "module_name"), + before_module_with_classes.ClassWithTwoMethods.my_method()).replace( + "before_module_with_classes", "module_name"), code_object_pickler._get_code_path( - after_module_with_classes.ClassWithTwoMethods.my_method() - ).replace("after_module_with_classes", "module_name"), + after_module_with_classes.ClassWithTwoMethods.my_method()).replace( + "after_module_with_classes", "module_name"), ) def test_identifiers_remove_function_in_class(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_classes.ClassWithTwoMethods.my_method() - ).replace("before_module_with_classes", "module_name"), + before_module_with_classes.ClassWithTwoMethods.my_method()).replace( + "before_module_with_classes", "module_name"), code_object_pickler._get_code_path( - after_module_with_classes.ClassWithTwoMethods.my_method() - ).replace("after_module_with_classes", "module_name"), + after_module_with_classes.ClassWithTwoMethods.my_method()).replace( + "after_module_with_classes", "module_name"), ) def test_identifiers_add_global_variable(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_functions.my_function() - ).replace("before_module_with_functions", "module_name"), + before_module_with_functions.my_function()).replace( + "before_module_with_functions", "module_name"), code_object_pickler._get_code_path( - after_module_with_global_variable.my_function() - ).replace("after_module_with_global_variable", "module_name"), + after_module_with_global_variable.my_function()).replace( + "after_module_with_global_variable", "module_name"), ) def test_identifiers_add_top_level_function(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_functions.my_function() - ).replace("before_module_with_functions", "module_name"), + before_module_with_functions.my_function()).replace( + "before_module_with_functions", "module_name"), code_object_pickler._get_code_path( - after_module_add_function.my_function() - ).replace("after_module_add_function", "module_name"), + after_module_add_function.my_function()).replace( + "after_module_add_function", "module_name"), ) def test_identifiers_add_local_variable_in_function(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_functions.my_function() - ).replace("before_module_with_functions", "module_name"), + before_module_with_functions.my_function()).replace( + "before_module_with_functions", "module_name"), code_object_pickler._get_code_path( - after_module_add_variable.my_function() - ).replace("after_module_add_variable", "module_name"), + after_module_add_variable.my_function()).replace( + "after_module_add_variable", "module_name"), ) def test_identifiers_remove_local_variable_in_function(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_functions.my_function() - ).replace("before_module_with_functions", "module_name"), + before_module_with_functions.my_function()).replace( + "before_module_with_functions", "module_name"), code_object_pickler._get_code_path( - after_module_remove_variable.my_function() - ).replace("after_module_remove_variable", "module_name"), + after_module_remove_variable.my_function()).replace( + "after_module_remove_variable", "module_name"), ) def test_identifiers_nested_function_in_function(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_functions.my_function() - ).replace("before_module_with_functions", "module_name"), + before_module_with_functions.my_function()).replace( + "before_module_with_functions", "module_name"), code_object_pickler._get_code_path( - after_module_with_nested_function.my_function() - ).replace("after_module_with_nested_function", "module_name"), + after_module_with_nested_function.my_function()).replace( + "after_module_with_nested_function", "module_name"), ) def test_identifiers_nested_function_2_in_function(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_functions.my_function() - ).replace("before_module_with_functions", "module_name"), + before_module_with_functions.my_function()).replace( + "before_module_with_functions", "module_name"), code_object_pickler._get_code_path( - after_module_with_nested_function_2.my_function() - ).replace("after_module_with_nested_function_2", "module_name"), + after_module_with_nested_function_2.my_function()).replace( + "after_module_with_nested_function_2", "module_name"), ) def test_identifiers_add_class(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_functions.my_function() - ).replace("before_module_with_functions", "module_name"), + before_module_with_functions.my_function()).replace( + "before_module_with_functions", "module_name"), code_object_pickler._get_code_path( - after_module_with_single_class.my_function() - ).replace("after_module_with_single_class", "module_name"), + after_module_with_single_class.my_function()).replace( + "after_module_with_single_class", "module_name"), ) def test_identifiers_add_lambda_variable_in_function(self): self.assertNotEqual( code_object_pickler._get_code_path( - before_module_with_functions.my_function() - ).replace("before_module_with_functions", "module_name"), + before_module_with_functions.my_function()).replace( + "before_module_with_functions", "module_name"), code_object_pickler._get_code_path( - after_module_add_lambda_variable.my_function() - ).replace("after_module_add_lambda_variable", "module_name"), + after_module_add_lambda_variable.my_function()).replace( + "after_module_add_lambda_variable", "module_name"), ) def test_identifiers_remove_lambda_variable_in_function(self): self.assertNotEqual( code_object_pickler._get_code_path( - before_module_with_lambdas.my_function() - ).replace("before_module_with_lambdas", "module_name"), + before_module_with_lambdas.my_function()).replace( + "before_module_with_lambdas", "module_name"), code_object_pickler._get_code_path( - after_module_remove_lambda_variable.my_function() - ).replace("after_module_remove_lambda_variable", "module_name"), + after_module_remove_lambda_variable.my_function()).replace( + "after_module_remove_lambda_variable", "module_name"), ) From e0e9b4b45cb639925ee37c8fa81446795fc1c187 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 16:52:14 -0400 Subject: [PATCH 045/142] Update after_module_with_nested_function.py --- .../internal/test_cases/after_module_with_nested_function.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py index 64ec6c0c6fe6..0cb288361ab2 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py @@ -21,7 +21,9 @@ def my_function(): a = 1 b = lambda: 2 + def nested_function(): c = 3 return c + return b From 556526e473f9bed5d7bc260d0f07d7fa4992b4e5 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 16:53:01 -0400 Subject: [PATCH 046/142] Update after_module_with_classes.py --- .../internal/test_cases/after_module_with_classes.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py index 1546c1c6e408..4e180b28a655 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py @@ -51,6 +51,7 @@ def my_method(): def nested_function(): c = 3 return c + a = 1 b = lambda: 2 return b @@ -60,9 +61,11 @@ class ClassWithNestedFunction2: def my_method(): a = 1 b = lambda: 2 + def nested_function(): c = 3 return c + return b From 0469dc8168868b12f6f0b8a2b171073753ddeead Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 22 Jul 2025 16:53:17 -0400 Subject: [PATCH 047/142] Update after_module_with_nested_function_2.py --- .../internal/test_cases/after_module_with_nested_function_2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function_2.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function_2.py index a14a0e1b6021..a158145a6e91 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function_2.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function_2.py @@ -22,6 +22,7 @@ def my_function(): def nested_function(): c = 3 return c + a = 1 b = lambda: 2 return b From 3c7f4cd5034043d76f8d6a4731cec3c126228e6f Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Wed, 23 Jul 2025 10:07:43 -0400 Subject: [PATCH 048/142] Update code_object_pickler_test.py --- sdks/python/apache_beam/internal/code_object_pickler_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index f2921610391f..a90beec4ed9c 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -69,7 +69,7 @@ def process(self): class ClassWithStaticMethod: @staticmethod - def static_method(): # pylint: disable=staticmethod-use + def static_method(): return 1 @@ -183,4 +183,3 @@ def test_roundtrip(self, callable, _): if __name__ == "__main__": unittest.main() - From 6fb63791c7eb2654c5a109001518125d1a701397 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Wed, 23 Jul 2025 10:08:03 -0400 Subject: [PATCH 049/142] Update code_object_pickler.py --- sdks/python/apache_beam/internal/code_object_pickler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler.py b/sdks/python/apache_beam/internal/code_object_pickler.py index 3e0d71377ec7..cd72d43e04f5 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler.py +++ b/sdks/python/apache_beam/internal/code_object_pickler.py @@ -18,7 +18,6 @@ import collections import hashlib import inspect -import os import re import sys import types From df6b1f50a76c2ed44b7de1bb4007df75c49beb1c Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Wed, 23 Jul 2025 10:11:32 -0400 Subject: [PATCH 050/142] Update pickle_code_path_test.py --- sdks/python/apache_beam/internal/pickle_code_path_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/internal/pickle_code_path_test.py b/sdks/python/apache_beam/internal/pickle_code_path_test.py index 8029fee708a8..ac0b4d439828 100644 --- a/sdks/python/apache_beam/internal/pickle_code_path_test.py +++ b/sdks/python/apache_beam/internal/pickle_code_path_test.py @@ -15,9 +15,10 @@ # limitations under the License. # -"""Tests for generating stable identifiers for Python __code__ to use for Pickle serialization.""" +"""Tests for generating stable identifiers to use for Pickle serialization.""" import unittest +import apache_beam.internal.test_cases from apache_beam.internal import code_object_pickler from apache_beam.internal.test_cases import after_module_add_function @@ -33,7 +34,6 @@ from apache_beam.internal.test_cases import before_module_with_classes from apache_beam.internal.test_cases import before_module_with_functions from apache_beam.internal.test_cases import before_module_with_lambdas -from apache_beam.internal.test_cases import module_with_default_argument class CodePathTest(unittest.TestCase): From b0f71d7e741b0179f3cb994561c9570a61e13476 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Wed, 23 Jul 2025 10:45:44 -0400 Subject: [PATCH 051/142] Update pickle_code_path_test.py --- .../internal/pickle_code_path_test.py | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/sdks/python/apache_beam/internal/pickle_code_path_test.py b/sdks/python/apache_beam/internal/pickle_code_path_test.py index ac0b4d439828..7600488452bb 100644 --- a/sdks/python/apache_beam/internal/pickle_code_path_test.py +++ b/sdks/python/apache_beam/internal/pickle_code_path_test.py @@ -48,22 +48,22 @@ def test_get_code_from_stable_reference_invalid_default_index(self): "function_with_lambda_default_argument.__defaults__[1]") def test_get_code_from_stable_reference_invalid_single_name_path(self): - with self.assertRaisesRegex( - AttributeError, "Could not find code object with path"): + with self.assertRaisesRegex(AttributeError, + "Could not find code object with path"): code_object_pickler._get_code_from_stable_reference( "apache_beam.internal.test_cases.before_module_with_lambdas." "my_function.__code__.co_consts[something]") def test_get_code_from_stable_reference_invalid_lambda_with_args_path(self): - with self.assertRaisesRegex( - AttributeError, "Could not find code object with path"): + with self.assertRaisesRegex(AttributeError, + "Could not find code object with path"): code_object_pickler._get_code_from_stable_reference( "apache_beam.internal.test_cases.before_module_with_lambdas." "my_function.__code__.co_consts[, ('x',)]") def test_get_code_from_stable_reference_invalid_lambda_with_hash_path(self): - with self.assertRaisesRegex( - AttributeError, "Could not find code object with path"): + with self.assertRaisesRegex(AttributeError, + "Could not find code object with path"): code_object_pickler._get_code_from_stable_reference( "apache_beam.internal.test_cases.before_module_with_lambdas." "my_function.__code__.co_consts[, ('',), 1234567890]") @@ -100,8 +100,7 @@ def test_get_code_remove_lambda_variable_in_class(self): code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( before_module_with_classes.RemoveLambdaVariable.my_method()). - replace("before_module_with_classes", "after_module_with_classes") - ) + replace("before_module_with_classes", "after_module_with_classes")) def test_get_code_nested_function_in_class(self): self.assertEqual( @@ -116,8 +115,9 @@ def test_get_code_nested_function_2_in_class(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_classes.ClassWithNestedFunction2.my_method()). - replace("before_module_with_classes", "after_module_with_classes")), + before_module_with_classes.ClassWithNestedFunction2.my_method() + ).replace( + "before_module_with_classes", "after_module_with_classes")), after_module_with_classes.ClassWithNestedFunction2.my_method().__code__, ) @@ -146,7 +146,7 @@ def test_get_code_add_global_variable(self): before_module_with_functions.my_function()).replace( "before_module_with_functions", "after_module_with_global_variable", - )), + )), after_module_with_global_variable.my_function().__code__, ) @@ -261,8 +261,8 @@ def test_identifiers_add_lambda_variable_in_class(self): def test_identifiers_remove_lambda_variable_in_class(self): self.assertNotEqual( code_object_pickler._get_code_path( - before_module_with_classes.RemoveLambdaVariable.my_method()).replace( - "before_module_with_classes", "module_name"), + before_module_with_classes.RemoveLambdaVariable.my_method()). + replace("before_module_with_classes", "module_name"), code_object_pickler._get_code_path( after_module_with_classes.RemoveLambdaVariable.my_method()).replace( "after_module_with_classes", "module_name"), @@ -271,21 +271,21 @@ def test_identifiers_remove_lambda_variable_in_class(self): def test_identifiers_nested_function_in_class(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_classes.ClassWithNestedFunction.my_method()).replace( - "before_module_with_classes", "module_name"), + before_module_with_classes.ClassWithNestedFunction.my_method() + ).replace("before_module_with_classes", "module_name"), code_object_pickler._get_code_path( after_module_with_classes.ClassWithNestedFunction.my_method()). - replace("after_module_with_classes", "module_name"), + replace("after_module_with_classes", "module_name"), ) def test_identifiers_nested_function_2_in_class(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_classes.ClassWithNestedFunction2.my_method()).replace( - "before_module_with_classes", "module_name"), + before_module_with_classes.ClassWithNestedFunction2.my_method() + ).replace("before_module_with_classes", "module_name"), code_object_pickler._get_code_path( after_module_with_classes.ClassWithNestedFunction2.my_method()). - replace("after_module_with_classes", "module_name"), + replace("after_module_with_classes", "module_name"), ) def test_identifiers_add_new_function_in_class(self): @@ -301,10 +301,10 @@ def test_identifiers_add_new_function_in_class(self): def test_identifiers_remove_function_in_class(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_classes.ClassWithTwoMethods.my_method()).replace( + before_module_with_classes.RemoveMethod.my_method()).replace( "before_module_with_classes", "module_name"), code_object_pickler._get_code_path( - after_module_with_classes.ClassWithTwoMethods.my_method()).replace( + after_module_with_classes.RemoveMethod.my_method()).replace( "after_module_with_classes", "module_name"), ) @@ -355,7 +355,7 @@ def test_identifiers_nested_function_in_function(self): "before_module_with_functions", "module_name"), code_object_pickler._get_code_path( after_module_with_nested_function.my_function()).replace( - "after_module_with_nested_function", "module_name"), + "after_module_with_nested_function", "module_name"), ) def test_identifiers_nested_function_2_in_function(self): @@ -365,7 +365,7 @@ def test_identifiers_nested_function_2_in_function(self): "before_module_with_functions", "module_name"), code_object_pickler._get_code_path( after_module_with_nested_function_2.my_function()).replace( - "after_module_with_nested_function_2", "module_name"), + "after_module_with_nested_function_2", "module_name"), ) def test_identifiers_add_class(self): @@ -375,7 +375,7 @@ def test_identifiers_add_class(self): "before_module_with_functions", "module_name"), code_object_pickler._get_code_path( after_module_with_single_class.my_function()).replace( - "after_module_with_single_class", "module_name"), + "after_module_with_single_class", "module_name"), ) def test_identifiers_add_lambda_variable_in_function(self): From 114b65dcd6023e23c4400802beba274d1b783111 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Wed, 23 Jul 2025 10:47:15 -0400 Subject: [PATCH 052/142] Update code_object_pickler.py --- sdks/python/apache_beam/internal/code_object_pickler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler.py b/sdks/python/apache_beam/internal/code_object_pickler.py index cd72d43e04f5..25cf92dc348f 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler.py +++ b/sdks/python/apache_beam/internal/code_object_pickler.py @@ -305,8 +305,7 @@ def _get_code_object_from_lambda_with_args_pattern( def _get_code_object_from_lambda_with_hash_pattern( - obj, lambda_with_hash_result, path -): + obj, lambda_with_hash_result, path): """Returns the code object from a lambda with hash pattern. Args: From cd3e33f2f90222770902a3a8cb69c4c3cc36ed13 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Wed, 23 Jul 2025 10:48:11 -0400 Subject: [PATCH 053/142] Update after_module_with_classes.py --- .../internal/test_cases/after_module_with_classes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py index 4e180b28a655..908e33e7ecc0 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py @@ -51,7 +51,7 @@ def my_method(): def nested_function(): c = 3 return c - + a = 1 b = lambda: 2 return b @@ -61,11 +61,11 @@ class ClassWithNestedFunction2: def my_method(): a = 1 b = lambda: 2 - + def nested_function(): c = 3 return c - + return b From 896c7b979371d55fc1ceba352199c3c6b7917caa Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Wed, 23 Jul 2025 10:48:35 -0400 Subject: [PATCH 054/142] Update after_module_with_nested_function.py --- .../internal/test_cases/after_module_with_nested_function.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py index 0cb288361ab2..10ba52c4d872 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py @@ -21,9 +21,9 @@ def my_function(): a = 1 b = lambda: 2 - + def nested_function(): c = 3 return c - + return b From 9d0003d4f20957db8bcb9536ddf99543dcf26fdc Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Wed, 23 Jul 2025 10:48:50 -0400 Subject: [PATCH 055/142] Update after_module_with_nested_function_2.py --- .../internal/test_cases/after_module_with_nested_function_2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function_2.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function_2.py index a158145a6e91..30744f0bbf89 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function_2.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function_2.py @@ -22,7 +22,7 @@ def my_function(): def nested_function(): c = 3 return c - + a = 1 b = lambda: 2 return b From 1db0215f11241cb1b7596221d2ca511e5838d4e6 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Wed, 23 Jul 2025 10:54:14 -0400 Subject: [PATCH 056/142] Update code_object_pickler_test.py --- sdks/python/apache_beam/internal/code_object_pickler_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index a90beec4ed9c..a7f7eec445c5 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -163,7 +163,7 @@ def get_lambda_from_dictionary(): ] -class DillTest(parameterized.TestCase): +class DillTest(unittest.TestCase): @parameterized.parameters(test_cases) def test_get_code_path(self, callable, expected): actual = code_object_pickler._get_code_path(callable) From cad4f8a5f4036dc84218f511c8faccd090542a52 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Wed, 23 Jul 2025 10:56:05 -0400 Subject: [PATCH 057/142] Update code_object_pickler_test.py --- sdks/python/apache_beam/internal/code_object_pickler_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index a7f7eec445c5..42dc4d3ebc74 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -18,7 +18,7 @@ import unittest from apache_beam.internal import code_object_pickler -from absl.testing import parameterized +from parameterized import parameterized def top_level_function(): From 5b00a85f38e57ff2970cc4eb55060af2c3dd9bae Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Wed, 23 Jul 2025 11:29:23 -0400 Subject: [PATCH 058/142] Update code_object_pickler_test.py --- .../python/apache_beam/internal/code_object_pickler_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index 42dc4d3ebc74..82ebfec69a4d 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -164,17 +164,17 @@ def get_lambda_from_dictionary(): class DillTest(unittest.TestCase): - @parameterized.parameters(test_cases) + @parameterized.expand(test_cases) def test_get_code_path(self, callable, expected): actual = code_object_pickler._get_code_path(callable) self.assertEqual(actual, expected) - @parameterized.parameters(test_cases) + @parameterized.expand(test_cases) def test_get_code_from_stable_reference(self, callable, path): actual = code_object_pickler._get_code_from_stable_reference(path) self.assertEqual(actual, callable.__code__) - @parameterized.parameters(test_cases) + @parameterized.expand(test_cases) def test_roundtrip(self, callable, _): path = code_object_pickler._get_code_path(callable) actual = code_object_pickler._get_code_from_stable_reference(path) From cd8e722a5cf56bc18124f4e1a53af2891072baba Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Wed, 23 Jul 2025 13:04:08 -0400 Subject: [PATCH 059/142] Update pickle_code_path_test.py --- sdks/python/apache_beam/internal/pickle_code_path_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/pickle_code_path_test.py b/sdks/python/apache_beam/internal/pickle_code_path_test.py index 7600488452bb..555a5d4b9466 100644 --- a/sdks/python/apache_beam/internal/pickle_code_path_test.py +++ b/sdks/python/apache_beam/internal/pickle_code_path_test.py @@ -292,7 +292,7 @@ def test_identifiers_add_new_function_in_class(self): self.assertEqual( code_object_pickler._get_code_path( before_module_with_classes.ClassWithTwoMethods.my_method()).replace( - "before_module_with_classes", "module_name"), + "before_module_with_classes", "module_name"), code_object_pickler._get_code_path( after_module_with_classes.ClassWithTwoMethods.my_method()).replace( "after_module_with_classes", "module_name"), From c18bf0401951faf05d010cd5a517a8b6675cdca1 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Wed, 23 Jul 2025 13:22:03 -0400 Subject: [PATCH 060/142] Update pickle_code_path_test.py --- sdks/python/apache_beam/internal/pickle_code_path_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/pickle_code_path_test.py b/sdks/python/apache_beam/internal/pickle_code_path_test.py index 555a5d4b9466..b7181e4dc458 100644 --- a/sdks/python/apache_beam/internal/pickle_code_path_test.py +++ b/sdks/python/apache_beam/internal/pickle_code_path_test.py @@ -18,7 +18,6 @@ """Tests for generating stable identifiers to use for Pickle serialization.""" import unittest -import apache_beam.internal.test_cases from apache_beam.internal import code_object_pickler from apache_beam.internal.test_cases import after_module_add_function From f77eff0dad9f341ca656f8b10f8cabcc191056ce Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Wed, 23 Jul 2025 16:22:52 -0400 Subject: [PATCH 061/142] Create __init__.py --- .../internal/test_cases/__init__.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 sdks/python/apache_beam/internal/test_cases/__init__.py diff --git a/sdks/python/apache_beam/internal/test_cases/__init__.py b/sdks/python/apache_beam/internal/test_cases/__init__.py new file mode 100644 index 000000000000..0bce5d68f724 --- /dev/null +++ b/sdks/python/apache_beam/internal/test_cases/__init__.py @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""For internal use only; no backwards-compatibility guarantees.""" From 3ff7cd54fdfefd7201cebe4640a2eee746b22069 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Wed, 23 Jul 2025 16:26:09 -0400 Subject: [PATCH 062/142] Update before_module_with_classes.py --- .../test_cases/before_module_with_classes.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/sdks/python/apache_beam/internal/test_cases/before_module_with_classes.py b/sdks/python/apache_beam/internal/test_cases/before_module_with_classes.py index cf9c60bde844..c9917ee1bede 100644 --- a/sdks/python/apache_beam/internal/test_cases/before_module_with_classes.py +++ b/sdks/python/apache_beam/internal/test_cases/before_module_with_classes.py @@ -19,61 +19,61 @@ class AddLocalVariable: - def my_method(): + def my_method(self): a = 1 b = lambda: 2 return b class RemoveLocalVariable: - def my_method(): + def my_method(self): a = 1 b = lambda: 2 return b class AddLambdaVariable: - def my_method(): + def my_method(self): a = 1 b = lambda: 2 return b class RemoveLambdaVariable: - def my_method(): + def my_method(self): a = lambda: 1 b = lambda: 2 return b class ClassWithNestedFunction: - def my_method(): + def my_method(self): a = 1 b = lambda: 2 return b class ClassWithNestedFunction2: - def my_method(): + def my_method(self): a = 1 b = lambda: 2 return b class ClassWithTwoMethods: - def my_method(): + def my_method(self): a = 1 b = lambda: 2 return b class RemoveMethod: - def another_method(): + def another_method(self): a = 1 b = lambda: 2 return b - def my_method(): + def my_method(self): a = 1 b = lambda: 2 return b From defa23c6631f408ac1736366048b91a2c97c7746 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Wed, 23 Jul 2025 16:27:33 -0400 Subject: [PATCH 063/142] Update after_module_with_single_class.py --- .../internal/test_cases/after_module_with_single_class.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_single_class.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_single_class.py index dab476321fa2..8bffdf99f1ae 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_with_single_class.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_single_class.py @@ -19,7 +19,7 @@ class MyClass: - def another_function(): + def another_function(self): a = 1 b = lambda: 2 return b From f2666c4b10896289d0a5b9b22715cd7fc6720e2c Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Wed, 23 Jul 2025 16:28:11 -0400 Subject: [PATCH 064/142] Update after_module_with_classes.py --- .../test_cases/after_module_with_classes.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py index 908e33e7ecc0..59f02579b87c 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py @@ -19,7 +19,7 @@ class AddLocalVariable: - def my_method(): + def my_method(self): a = 1 b = lambda: 2 new_local_variable = 3 @@ -27,13 +27,13 @@ def my_method(): class RemoveLocalVariable: - def my_method(): + def my_method(self): b = lambda: 2 return b class AddLambdaVariable: - def my_method(): + def my_method(self): a = 1 b = lambda: 2 c = lambda: 3 @@ -41,13 +41,13 @@ def my_method(): class RemoveLambdaVariable: - def my_method(): + def my_method(self): b = lambda: 2 return b class ClassWithNestedFunction: - def my_method(): + def my_method(self): def nested_function(): c = 3 return c @@ -58,7 +58,7 @@ def nested_function(): class ClassWithNestedFunction2: - def my_method(): + def my_method(self): a = 1 b = lambda: 2 @@ -70,19 +70,19 @@ def nested_function(): class ClassWithTwoMethods: - def another_method(): + def another_method(self): a = 1 b = lambda: 2 return b - def my_method(): + def my_method(self): a = 1 b = lambda: 2 return b class RemoveMethod: - def my_method(): + def my_method(self): a = 1 b = lambda: 2 From b92abbba814769af102bacffa7cbda435845c0f4 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Wed, 23 Jul 2025 16:51:27 -0400 Subject: [PATCH 065/142] Update pickle_code_path_test.py --- .../internal/pickle_code_path_test.py | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/sdks/python/apache_beam/internal/pickle_code_path_test.py b/sdks/python/apache_beam/internal/pickle_code_path_test.py index b7181e4dc458..f322cb7b76ad 100644 --- a/sdks/python/apache_beam/internal/pickle_code_path_test.py +++ b/sdks/python/apache_beam/internal/pickle_code_path_test.py @@ -71,7 +71,7 @@ def test_get_code_add_local_variable_in_class(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_classes.AddLocalVariable.my_method()). + before_module_with_classes.AddLocalVariable.my_method(self)). replace("before_module_with_classes", "after_module_with_classes")), after_module_with_classes.AddLocalVariable.my_method().__code__, ) @@ -80,7 +80,7 @@ def test_get_code_remove_local_variable_in_class(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_classes.RemoveLocalVariable.my_method()). + before_module_with_classes.RemoveLocalVariable.my_method(self)). replace("before_module_with_classes", "after_module_with_classes")), after_module_with_classes.RemoveLocalVariable.my_method().__code__, ) @@ -89,7 +89,7 @@ def test_get_code_add_lambda_variable_in_class(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_classes.AddLambdaVariable.my_method()). + before_module_with_classes.AddLambdaVariable.my_method(self)). replace("before_module_with_classes", "after_module_with_classes")), after_module_with_classes.AddLambdaVariable.my_method().__code__, ) @@ -98,14 +98,14 @@ def test_get_code_remove_lambda_variable_in_class(self): with self.assertRaisesRegex(AttributeError, "object has no attribute"): code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_classes.RemoveLambdaVariable.my_method()). + before_module_with_classes.RemoveLambdaVariable.my_method(self)). replace("before_module_with_classes", "after_module_with_classes")) def test_get_code_nested_function_in_class(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_classes.ClassWithNestedFunction.my_method()). + before_module_with_classes.ClassWithNestedFunction.my_method(self)). replace("before_module_with_classes", "after_module_with_classes")), after_module_with_classes.ClassWithNestedFunction.my_method().__code__, ) @@ -114,7 +114,7 @@ def test_get_code_nested_function_2_in_class(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_classes.ClassWithNestedFunction2.my_method() + before_module_with_classes.ClassWithNestedFunction2.my_method(self) ).replace( "before_module_with_classes", "after_module_with_classes")), after_module_with_classes.ClassWithNestedFunction2.my_method().__code__, @@ -124,7 +124,7 @@ def test_get_code_add_new_function_in_class(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_classes.ClassWithTwoMethods.my_method()). + before_module_with_classes.ClassWithTwoMethods.my_method(self)). replace("before_module_with_classes", "after_module_with_classes")), after_module_with_classes.ClassWithTwoMethods.my_method().__code__, ) @@ -133,7 +133,7 @@ def test_get_code_remove_method_in_class(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_classes.RemoveMethod.my_method()).replace( + before_module_with_classes.RemoveMethod.my_method(self)).replace( "before_module_with_classes", "after_module_with_classes")), after_module_with_classes.RemoveMethod.my_method().__code__, ) @@ -230,80 +230,80 @@ def test_get_code_remove_lambda_variable_in_function(self): def test_identifiers_new_local_variable_in_class(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_classes.AddLocalVariable.my_method()).replace( + before_module_with_classes.AddLocalVariable.my_method(self)).replace( "before_module_with_classes", "module_name"), code_object_pickler._get_code_path( - after_module_with_classes.AddLocalVariable.my_method()).replace( + after_module_with_classes.AddLocalVariable.my_method(self)).replace( "after_module_with_classes", "module_name"), ) def test_identifiers_remove_local_variable_in_class(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_classes.RemoveLocalVariable.my_method()).replace( + before_module_with_classes.RemoveLocalVariable.my_method(self)).replace( "before_module_with_classes", "module_name"), code_object_pickler._get_code_path( - after_module_with_classes.RemoveLocalVariable.my_method()).replace( + after_module_with_classes.RemoveLocalVariable.my_method(self)).replace( "after_module_with_classes", "module_name"), ) def test_identifiers_add_lambda_variable_in_class(self): self.assertNotEqual( code_object_pickler._get_code_path( - before_module_with_classes.AddLambdaVariable.my_method()).replace( + before_module_with_classes.AddLambdaVariable.my_method(self)).replace( "before_module_with_classes", "module_name"), code_object_pickler._get_code_path( - after_module_with_classes.AddLambdaVariable.my_method()).replace( + after_module_with_classes.AddLambdaVariable.my_method(self)).replace( "after_module_with_classes", "module_name"), ) def test_identifiers_remove_lambda_variable_in_class(self): self.assertNotEqual( code_object_pickler._get_code_path( - before_module_with_classes.RemoveLambdaVariable.my_method()). + before_module_with_classes.RemoveLambdaVariable.my_method(self)). replace("before_module_with_classes", "module_name"), code_object_pickler._get_code_path( - after_module_with_classes.RemoveLambdaVariable.my_method()).replace( + after_module_with_classes.RemoveLambdaVariable.my_method(self)).replace( "after_module_with_classes", "module_name"), ) def test_identifiers_nested_function_in_class(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_classes.ClassWithNestedFunction.my_method() + before_module_with_classes.ClassWithNestedFunction.my_method(self) ).replace("before_module_with_classes", "module_name"), code_object_pickler._get_code_path( - after_module_with_classes.ClassWithNestedFunction.my_method()). + after_module_with_classes.ClassWithNestedFunction.my_method(self)). replace("after_module_with_classes", "module_name"), ) def test_identifiers_nested_function_2_in_class(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_classes.ClassWithNestedFunction2.my_method() + before_module_with_classes.ClassWithNestedFunction2.my_method(self) ).replace("before_module_with_classes", "module_name"), code_object_pickler._get_code_path( - after_module_with_classes.ClassWithNestedFunction2.my_method()). + after_module_with_classes.ClassWithNestedFunction2.my_method(self)). replace("after_module_with_classes", "module_name"), ) def test_identifiers_add_new_function_in_class(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_classes.ClassWithTwoMethods.my_method()).replace( + before_module_with_classes.ClassWithTwoMethods.my_method(self)).replace( "before_module_with_classes", "module_name"), code_object_pickler._get_code_path( - after_module_with_classes.ClassWithTwoMethods.my_method()).replace( + after_module_with_classes.ClassWithTwoMethods.my_method(self)).replace( "after_module_with_classes", "module_name"), ) def test_identifiers_remove_function_in_class(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_classes.RemoveMethod.my_method()).replace( + before_module_with_classes.RemoveMethod.my_method(self)).replace( "before_module_with_classes", "module_name"), code_object_pickler._get_code_path( - after_module_with_classes.RemoveMethod.my_method()).replace( + after_module_with_classes.RemoveMethod.my_method(self)).replace( "after_module_with_classes", "module_name"), ) From d2521438f7eab688040b3edafaacce02d7dcff4f Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Wed, 23 Jul 2025 16:53:41 -0400 Subject: [PATCH 066/142] Update code_object_pickler_test.py --- .../internal/code_object_pickler_test.py | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index 82ebfec69a4d..be09120a2f12 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -108,58 +108,58 @@ def get_lambda_from_dictionary(): test_cases = [ - (top_level_function, "__main__.top_level_function.__code__"), - (top_level_lambda, "__main__.top_level_lambda.__code__"), + (top_level_function, "apache_beam.internal.code_object_pickler_test.top_level_function.__code__"), + (top_level_lambda, "apache_beam.internal.code_object_pickler_test.top_level_lambda.__code__"), ( get_nested_function(), - ("__main__.get_nested_function.__code__.co_consts" + ("apache_beam.internal.code_object_pickler_test.get_nested_function.__code__.co_consts" "[nested_function]")), ( get_lambda_from_dictionary(), ( - "__main__.get_lambda_from_dictionary.__code__.co_consts" + "apache_beam.internal.code_object_pickler_test.get_lambda_from_dictionary.__code__.co_consts" "[, ('x',)]")), ( get_lambda_from_dictionary_same_args(), ( - "__main__.get_lambda_from_dictionary_same_args.__code__.co_consts" + "apache_beam.internal.code_object_pickler_test.get_lambda_from_dictionary_same_args.__code__.co_consts" "[, ('x',), " "ac455edc80140a7f12ad229519456f71]")), ( function_with_lambda_default_argument(), ( - "__main__.function_with_lambda_default_argument.__defaults__[0]" + "apache_beam.internal.code_object_pickler_test.function_with_lambda_default_argument.__defaults__[0]" ".__code__")), ( function_with_function_default_argument(), - "__main__.top_level_function.__code__"), - (add_one, "__main__.function_decorator.__code__.co_consts[]"), - (ClassWithFunction.process, "__main__.ClassWithFunction.process.__code__"), + "apache_beam.internal.code_object_pickler_test.top_level_function.__code__"), + (add_one, "apache_beam.internal.code_object_pickler_test.function_decorator.__code__.co_consts[]"), + (ClassWithFunction.process, "apache_beam.internal.code_object_pickler_test.ClassWithFunction.process.__code__"), ( ClassWithStaticMethod.static_method, - "__main__.ClassWithStaticMethod.static_method.__code__"), + "apache_beam.internal.code_object_pickler_test.ClassWithStaticMethod.static_method.__code__"), ( ClassWithClassMethod.class_method, - "__main__.ClassWithClassMethod.class_method.__code__"), + "apache_beam.internal.code_object_pickler_test.ClassWithClassMethod.class_method.__code__"), ( ClassWithNestedFunction().process(), ( - "__main__.ClassWithNestedFunction.process.__code__.co_consts" + "apache_beam.internal.code_object_pickler_test.ClassWithNestedFunction.process.__code__.co_consts" "[nested_function]")), ( ClassWithLambda().process(), - "__main__.ClassWithLambda.process.__code__.co_consts[]"), + "apache_beam.internal.code_object_pickler_test.ClassWithLambda.process.__code__.co_consts[]"), ( ClassWithNestedClass.InnerClass().process, - "__main__.ClassWithNestedClass.InnerClass.process.__code__"), + "apache_beam.internal.code_object_pickler_test.ClassWithNestedClass.InnerClass.process.__code__"), ( ClassWithNestedLambda().process(), ( - "__main__.ClassWithNestedLambda.process.__code__.co_consts" + "apache_beam.internal.code_object_pickler_test.ClassWithNestedLambda.process.__code__.co_consts" "[get_lambda_from_dictionary].co_consts[, ('x',)]")), ( ClassWithNestedLambda.process, - "__main__.ClassWithNestedLambda.process.__code__"), + "apache_beam.internal.code_object_pickler_test.ClassWithNestedLambda.process.__code__"), ] From 94e8dbb45cc1fcc5f62bef3afe2b41f7dbd377cf Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Thu, 24 Jul 2025 10:21:16 -0400 Subject: [PATCH 067/142] Update code_object_pickler_test.py --- .../internal/code_object_pickler_test.py | 40 ++++++++++++++----- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index be09120a2f12..e2d315d9e68d 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -108,12 +108,19 @@ def get_lambda_from_dictionary(): test_cases = [ - (top_level_function, "apache_beam.internal.code_object_pickler_test.top_level_function.__code__"), - (top_level_lambda, "apache_beam.internal.code_object_pickler_test.top_level_lambda.__code__"), + ( + top_level_function, + "apache_beam.internal.code_object_pickler_test.top_level_function.__code__" + ), + ( + top_level_lambda, + "apache_beam.internal.code_object_pickler_test.top_level_lambda.__code__" + ), ( get_nested_function(), - ("apache_beam.internal.code_object_pickler_test.get_nested_function.__code__.co_consts" - "[nested_function]")), + ( + "apache_beam.internal.code_object_pickler_test.get_nested_function.__code__.co_consts" + "[nested_function]")), ( get_lambda_from_dictionary(), ( @@ -132,15 +139,24 @@ def get_lambda_from_dictionary(): ".__code__")), ( function_with_function_default_argument(), - "apache_beam.internal.code_object_pickler_test.top_level_function.__code__"), - (add_one, "apache_beam.internal.code_object_pickler_test.function_decorator.__code__.co_consts[]"), - (ClassWithFunction.process, "apache_beam.internal.code_object_pickler_test.ClassWithFunction.process.__code__"), + "apache_beam.internal.code_object_pickler_test.top_level_function.__code__" + ), + ( + add_one, + "apache_beam.internal.code_object_pickler_test.function_decorator.__code__.co_consts[]" + ), + ( + ClassWithFunction.process, + "apache_beam.internal.code_object_pickler_test.ClassWithFunction.process.__code__" + ), ( ClassWithStaticMethod.static_method, - "apache_beam.internal.code_object_pickler_test.ClassWithStaticMethod.static_method.__code__"), + "apache_beam.internal.code_object_pickler_test.ClassWithStaticMethod.static_method.__code__" + ), ( ClassWithClassMethod.class_method, - "apache_beam.internal.code_object_pickler_test.ClassWithClassMethod.class_method.__code__"), + "apache_beam.internal.code_object_pickler_test.ClassWithClassMethod.class_method.__code__" + ), ( ClassWithNestedFunction().process(), ( @@ -148,7 +164,8 @@ def get_lambda_from_dictionary(): "[nested_function]")), ( ClassWithLambda().process(), - "apache_beam.internal.code_object_pickler_test.ClassWithLambda.process.__code__.co_consts[]"), + "apache_beam.internal.code_object_pickler_test.ClassWithLambda.process.__code__.co_consts[]" + ), ( ClassWithNestedClass.InnerClass().process, "apache_beam.internal.code_object_pickler_test.ClassWithNestedClass.InnerClass.process.__code__"), @@ -159,7 +176,8 @@ def get_lambda_from_dictionary(): "[get_lambda_from_dictionary].co_consts[, ('x',)]")), ( ClassWithNestedLambda.process, - "apache_beam.internal.code_object_pickler_test.ClassWithNestedLambda.process.__code__"), + "apache_beam.internal.code_object_pickler_test.ClassWithNestedLambda.process.__code__" + ), ] From d417145e65fbfffbe9c3e4162187d311cab22906 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Thu, 24 Jul 2025 10:31:45 -0400 Subject: [PATCH 068/142] Update pickle_code_path_test.py --- .../internal/pickle_code_path_test.py | 101 ++++++++++-------- 1 file changed, 58 insertions(+), 43 deletions(-) diff --git a/sdks/python/apache_beam/internal/pickle_code_path_test.py b/sdks/python/apache_beam/internal/pickle_code_path_test.py index f322cb7b76ad..491a50726d93 100644 --- a/sdks/python/apache_beam/internal/pickle_code_path_test.py +++ b/sdks/python/apache_beam/internal/pickle_code_path_test.py @@ -71,8 +71,10 @@ def test_get_code_add_local_variable_in_class(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_classes.AddLocalVariable.my_method(self)). - replace("before_module_with_classes", "after_module_with_classes")), + before_module_with_classes.AddLocalVariable.my_method( + self)).replace( + "before_module_with_classes", + "after_module_with_classes")), after_module_with_classes.AddLocalVariable.my_method().__code__, ) @@ -80,8 +82,10 @@ def test_get_code_remove_local_variable_in_class(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_classes.RemoveLocalVariable.my_method(self)). - replace("before_module_with_classes", "after_module_with_classes")), + before_module_with_classes.RemoveLocalVariable.my_method( + self)).replace( + "before_module_with_classes", + "after_module_with_classes")), after_module_with_classes.RemoveLocalVariable.my_method().__code__, ) @@ -89,8 +93,10 @@ def test_get_code_add_lambda_variable_in_class(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_classes.AddLambdaVariable.my_method(self)). - replace("before_module_with_classes", "after_module_with_classes")), + before_module_with_classes.AddLambdaVariable.my_method( + self)).replace( + "before_module_with_classes", + "after_module_with_classes")), after_module_with_classes.AddLambdaVariable.my_method().__code__, ) @@ -98,15 +104,19 @@ def test_get_code_remove_lambda_variable_in_class(self): with self.assertRaisesRegex(AttributeError, "object has no attribute"): code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_classes.RemoveLambdaVariable.my_method(self)). - replace("before_module_with_classes", "after_module_with_classes")) + before_module_with_classes.RemoveLambdaVariable.my_method( + self)).replace( + "before_module_with_classes", + "after_module_with_classes")) def test_get_code_nested_function_in_class(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_classes.ClassWithNestedFunction.my_method(self)). - replace("before_module_with_classes", "after_module_with_classes")), + before_module_with_classes.ClassWithNestedFunction.my_method( + self)).replace( + "before_module_with_classes", + "after_module_with_classes")), after_module_with_classes.ClassWithNestedFunction.my_method().__code__, ) @@ -114,9 +124,10 @@ def test_get_code_nested_function_2_in_class(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_classes.ClassWithNestedFunction2.my_method(self) - ).replace( - "before_module_with_classes", "after_module_with_classes")), + before_module_with_classes.ClassWithNestedFunction2.my_method( + self)).replace( + "before_module_with_classes", + "after_module_with_classes")), after_module_with_classes.ClassWithNestedFunction2.my_method().__code__, ) @@ -124,8 +135,10 @@ def test_get_code_add_new_function_in_class(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_classes.ClassWithTwoMethods.my_method(self)). - replace("before_module_with_classes", "after_module_with_classes")), + before_module_with_classes.ClassWithTwoMethods.my_method( + self)).replace( + "before_module_with_classes", + "after_module_with_classes")), after_module_with_classes.ClassWithTwoMethods.my_method().__code__, ) @@ -133,8 +146,10 @@ def test_get_code_remove_method_in_class(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( - before_module_with_classes.RemoveMethod.my_method(self)).replace( - "before_module_with_classes", "after_module_with_classes")), + before_module_with_classes.RemoveMethod.my_method( + self)).replace( + "before_module_with_classes", + "after_module_with_classes")), after_module_with_classes.RemoveMethod.my_method().__code__, ) @@ -230,8 +245,8 @@ def test_get_code_remove_lambda_variable_in_function(self): def test_identifiers_new_local_variable_in_class(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_classes.AddLocalVariable.my_method(self)).replace( - "before_module_with_classes", "module_name"), + before_module_with_classes.AddLocalVariable.my_method( + self)).replace("before_module_with_classes", "module_name"), code_object_pickler._get_code_path( after_module_with_classes.AddLocalVariable.my_method(self)).replace( "after_module_with_classes", "module_name"), @@ -240,61 +255,61 @@ def test_identifiers_new_local_variable_in_class(self): def test_identifiers_remove_local_variable_in_class(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_classes.RemoveLocalVariable.my_method(self)).replace( - "before_module_with_classes", "module_name"), + before_module_with_classes.RemoveLocalVariable.my_method( + self)).replace("before_module_with_classes", "module_name"), code_object_pickler._get_code_path( - after_module_with_classes.RemoveLocalVariable.my_method(self)).replace( - "after_module_with_classes", "module_name"), + after_module_with_classes.RemoveLocalVariable.my_method( + self)).replace("after_module_with_classes", "module_name"), ) def test_identifiers_add_lambda_variable_in_class(self): self.assertNotEqual( code_object_pickler._get_code_path( - before_module_with_classes.AddLambdaVariable.my_method(self)).replace( - "before_module_with_classes", "module_name"), + before_module_with_classes.AddLambdaVariable.my_method( + self)).replace("before_module_with_classes", "module_name"), code_object_pickler._get_code_path( - after_module_with_classes.AddLambdaVariable.my_method(self)).replace( - "after_module_with_classes", "module_name"), + after_module_with_classes.AddLambdaVariable.my_method( + self)).replace("after_module_with_classes", "module_name"), ) def test_identifiers_remove_lambda_variable_in_class(self): self.assertNotEqual( code_object_pickler._get_code_path( - before_module_with_classes.RemoveLambdaVariable.my_method(self)). - replace("before_module_with_classes", "module_name"), + before_module_with_classes.RemoveLambdaVariable.my_method( + self)).replace("before_module_with_classes", "module_name"), code_object_pickler._get_code_path( - after_module_with_classes.RemoveLambdaVariable.my_method(self)).replace( - "after_module_with_classes", "module_name"), + after_module_with_classes.RemoveLambdaVariable.my_method( + self)).replace("after_module_with_classes", "module_name"), ) def test_identifiers_nested_function_in_class(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_classes.ClassWithNestedFunction.my_method(self) - ).replace("before_module_with_classes", "module_name"), + before_module_with_classes.ClassWithNestedFunction.my_method( + self)).replace("before_module_with_classes", "module_name"), code_object_pickler._get_code_path( - after_module_with_classes.ClassWithNestedFunction.my_method(self)). - replace("after_module_with_classes", "module_name"), + after_module_with_classes.ClassWithNestedFunction.my_method( + self)).replace("after_module_with_classes", "module_name"), ) def test_identifiers_nested_function_2_in_class(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_classes.ClassWithNestedFunction2.my_method(self) - ).replace("before_module_with_classes", "module_name"), + before_module_with_classes.ClassWithNestedFunction2.my_method( + self)).replace("before_module_with_classes", "module_name"), code_object_pickler._get_code_path( - after_module_with_classes.ClassWithNestedFunction2.my_method(self)). - replace("after_module_with_classes", "module_name"), + after_module_with_classes.ClassWithNestedFunction2.my_method( + self)).replace("after_module_with_classes", "module_name"), ) def test_identifiers_add_new_function_in_class(self): self.assertEqual( code_object_pickler._get_code_path( - before_module_with_classes.ClassWithTwoMethods.my_method(self)).replace( - "before_module_with_classes", "module_name"), + before_module_with_classes.ClassWithTwoMethods.my_method( + self)).replace("before_module_with_classes", "module_name"), code_object_pickler._get_code_path( - after_module_with_classes.ClassWithTwoMethods.my_method(self)).replace( - "after_module_with_classes", "module_name"), + after_module_with_classes.ClassWithTwoMethods.my_method( + self)).replace("after_module_with_classes", "module_name"), ) def test_identifiers_remove_function_in_class(self): From f3ec2f310adedc6ec77480ca97c0afafceafe30f Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Thu, 24 Jul 2025 10:57:33 -0400 Subject: [PATCH 069/142] Update code_object_pickler_test.py --- sdks/python/apache_beam/internal/code_object_pickler_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index e2d315d9e68d..0657e080957e 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -131,7 +131,7 @@ def get_lambda_from_dictionary(): ( "apache_beam.internal.code_object_pickler_test.get_lambda_from_dictionary_same_args.__code__.co_consts" "[, ('x',), " - "ac455edc80140a7f12ad229519456f71]")), + "a484a2b05a3ad69b07c7239c3c69c290]")), ( function_with_lambda_default_argument(), ( From c2c4baf9398ca669b51f46a8786593df4ae0f88c Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Thu, 24 Jul 2025 11:09:50 -0400 Subject: [PATCH 070/142] Update pickle_code_path_test.py --- .../internal/pickle_code_path_test.py | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/sdks/python/apache_beam/internal/pickle_code_path_test.py b/sdks/python/apache_beam/internal/pickle_code_path_test.py index 491a50726d93..a33b476f0466 100644 --- a/sdks/python/apache_beam/internal/pickle_code_path_test.py +++ b/sdks/python/apache_beam/internal/pickle_code_path_test.py @@ -33,6 +33,7 @@ from apache_beam.internal.test_cases import before_module_with_classes from apache_beam.internal.test_cases import before_module_with_functions from apache_beam.internal.test_cases import before_module_with_lambdas +from apache_beam.internal.test_cases import module_with_default_argument class CodePathTest(unittest.TestCase): @@ -75,7 +76,7 @@ def test_get_code_add_local_variable_in_class(self): self)).replace( "before_module_with_classes", "after_module_with_classes")), - after_module_with_classes.AddLocalVariable.my_method().__code__, + after_module_with_classes.AddLocalVariable.my_method(self).__code__, ) def test_get_code_remove_local_variable_in_class(self): @@ -86,7 +87,7 @@ def test_get_code_remove_local_variable_in_class(self): self)).replace( "before_module_with_classes", "after_module_with_classes")), - after_module_with_classes.RemoveLocalVariable.my_method().__code__, + after_module_with_classes.RemoveLocalVariable.my_method(self).__code__, ) def test_get_code_add_lambda_variable_in_class(self): @@ -97,7 +98,7 @@ def test_get_code_add_lambda_variable_in_class(self): self)).replace( "before_module_with_classes", "after_module_with_classes")), - after_module_with_classes.AddLambdaVariable.my_method().__code__, + after_module_with_classes.AddLambdaVariable.my_method(self).__code__, ) def test_get_code_remove_lambda_variable_in_class(self): @@ -117,7 +118,7 @@ def test_get_code_nested_function_in_class(self): self)).replace( "before_module_with_classes", "after_module_with_classes")), - after_module_with_classes.ClassWithNestedFunction.my_method().__code__, + after_module_with_classes.ClassWithNestedFunction.my_method(self).__code__, ) def test_get_code_nested_function_2_in_class(self): @@ -128,7 +129,7 @@ def test_get_code_nested_function_2_in_class(self): self)).replace( "before_module_with_classes", "after_module_with_classes")), - after_module_with_classes.ClassWithNestedFunction2.my_method().__code__, + after_module_with_classes.ClassWithNestedFunction2.my_method(self).__code__, ) def test_get_code_add_new_function_in_class(self): @@ -139,7 +140,7 @@ def test_get_code_add_new_function_in_class(self): self)).replace( "before_module_with_classes", "after_module_with_classes")), - after_module_with_classes.ClassWithTwoMethods.my_method().__code__, + after_module_with_classes.ClassWithTwoMethods.my_method(self).__code__, ) def test_get_code_remove_method_in_class(self): @@ -147,10 +148,10 @@ def test_get_code_remove_method_in_class(self): code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( before_module_with_classes.RemoveMethod.my_method( - self)).replace( - "before_module_with_classes", - "after_module_with_classes")), - after_module_with_classes.RemoveMethod.my_method().__code__, + self)).replace( + "before_module_with_classes", + "after_module_with_classes")), + after_module_with_classes.RemoveMethod.my_method(self).__code__, ) def test_get_code_add_global_variable(self): @@ -276,7 +277,7 @@ def test_identifiers_remove_lambda_variable_in_class(self): self.assertNotEqual( code_object_pickler._get_code_path( before_module_with_classes.RemoveLambdaVariable.my_method( - self)).replace("before_module_with_classes", "module_name"), + self)).replace("before_module_with_classes", "module_name"), code_object_pickler._get_code_path( after_module_with_classes.RemoveLambdaVariable.my_method( self)).replace("after_module_with_classes", "module_name"), @@ -306,10 +307,10 @@ def test_identifiers_add_new_function_in_class(self): self.assertEqual( code_object_pickler._get_code_path( before_module_with_classes.ClassWithTwoMethods.my_method( - self)).replace("before_module_with_classes", "module_name"), + self)).replace("before_module_with_classes", "module_name"), code_object_pickler._get_code_path( after_module_with_classes.ClassWithTwoMethods.my_method( - self)).replace("after_module_with_classes", "module_name"), + self)).replace("after_module_with_classes", "module_name"), ) def test_identifiers_remove_function_in_class(self): From 08a421e8a5cfb4beed164b70a1c15463a006044c Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Thu, 24 Jul 2025 13:52:29 -0400 Subject: [PATCH 071/142] Update code_object_pickler_test.py --- .../internal/code_object_pickler_test.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index 0657e080957e..112e1b1db45f 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -40,11 +40,6 @@ def get_lambda_from_dictionary(): return d["a"] -def get_lambda_from_dictionary_same_args(): - d = {"a": lambda x: 1, "b": lambda x: x + 1} - return d["a"] - - def function_with_lambda_default_argument(fn=lambda x: 1): return fn @@ -126,12 +121,6 @@ def get_lambda_from_dictionary(): ( "apache_beam.internal.code_object_pickler_test.get_lambda_from_dictionary.__code__.co_consts" "[, ('x',)]")), - ( - get_lambda_from_dictionary_same_args(), - ( - "apache_beam.internal.code_object_pickler_test.get_lambda_from_dictionary_same_args.__code__.co_consts" - "[, ('x',), " - "a484a2b05a3ad69b07c7239c3c69c290]")), ( function_with_lambda_default_argument(), ( @@ -168,7 +157,8 @@ def get_lambda_from_dictionary(): ), ( ClassWithNestedClass.InnerClass().process, - "apache_beam.internal.code_object_pickler_test.ClassWithNestedClass.InnerClass.process.__code__"), + "apache_beam.internal.code_object_pickler_test.ClassWithNestedClass.InnerClass.process.__code__" + ), ( ClassWithNestedLambda().process(), ( From 3c845632668a4783a18c9a812713f33d21a28808 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Thu, 24 Jul 2025 13:53:04 -0400 Subject: [PATCH 072/142] Update pickle_code_path_test.py --- sdks/python/apache_beam/internal/pickle_code_path_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/internal/pickle_code_path_test.py b/sdks/python/apache_beam/internal/pickle_code_path_test.py index a33b476f0466..ec7ed0e87c28 100644 --- a/sdks/python/apache_beam/internal/pickle_code_path_test.py +++ b/sdks/python/apache_beam/internal/pickle_code_path_test.py @@ -118,7 +118,8 @@ def test_get_code_nested_function_in_class(self): self)).replace( "before_module_with_classes", "after_module_with_classes")), - after_module_with_classes.ClassWithNestedFunction.my_method(self).__code__, + after_module_with_classes.ClassWithNestedFunction.my_method( + self).__code__, ) def test_get_code_nested_function_2_in_class(self): @@ -129,7 +130,8 @@ def test_get_code_nested_function_2_in_class(self): self)).replace( "before_module_with_classes", "after_module_with_classes")), - after_module_with_classes.ClassWithNestedFunction2.my_method(self).__code__, + after_module_with_classes.ClassWithNestedFunction2.my_method( + self).__code__, ) def test_get_code_add_new_function_in_class(self): From 393091d7d7d218ad3d432615c5e497d0109d8b58 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Fri, 25 Jul 2025 13:20:56 -0400 Subject: [PATCH 073/142] Update code_object_pickler_test.py --- .../internal/code_object_pickler_test.py | 50 ++++++++++++------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index 112e1b1db45f..c9cf1fe428da 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -105,68 +105,80 @@ def get_lambda_from_dictionary(): test_cases = [ ( top_level_function, - "apache_beam.internal.code_object_pickler_test.top_level_function.__code__" + "apache_beam.internal.code_object_pickler_test.top_level_function" + ".__code__" ), ( top_level_lambda, - "apache_beam.internal.code_object_pickler_test.top_level_lambda.__code__" + "apache_beam.internal.code_object_pickler_test.top_level_lambda" + ".__code__" ), ( get_nested_function(), ( - "apache_beam.internal.code_object_pickler_test.get_nested_function.__code__.co_consts" - "[nested_function]")), + "apache_beam.internal.code_object_pickler_test.get_nested_function" + ".__code__.co_consts[nested_function]")), ( get_lambda_from_dictionary(), ( - "apache_beam.internal.code_object_pickler_test.get_lambda_from_dictionary.__code__.co_consts" - "[, ('x',)]")), + "apache_beam.internal.code_object_pickler_test" + ".get_lambda_from_dictionary.__code__.co_consts[, ('x',)]")), ( function_with_lambda_default_argument(), ( - "apache_beam.internal.code_object_pickler_test.function_with_lambda_default_argument.__defaults__[0]" - ".__code__")), + "apache_beam.internal.code_object_pickler_test" + ".function_with_lambda_default_argument.__defaults__[0].__code__")), ( function_with_function_default_argument(), - "apache_beam.internal.code_object_pickler_test.top_level_function.__code__" + "apache_beam.internal.code_object_pickler_test.top_level_function" + ".__code__" ), ( add_one, - "apache_beam.internal.code_object_pickler_test.function_decorator.__code__.co_consts[]" + "apache_beam.internal.code_object_pickler_test.function_decorator" + ".__code__.co_consts[]" ), ( ClassWithFunction.process, - "apache_beam.internal.code_object_pickler_test.ClassWithFunction.process.__code__" + "apache_beam.internal.code_object_pickler_test.ClassWithFunction.process" + ".__code__" ), ( ClassWithStaticMethod.static_method, - "apache_beam.internal.code_object_pickler_test.ClassWithStaticMethod.static_method.__code__" + "apache_beam.internal.code_object_pickler_test.ClassWithStaticMethod" + ".static_method.__code__" ), ( ClassWithClassMethod.class_method, - "apache_beam.internal.code_object_pickler_test.ClassWithClassMethod.class_method.__code__" + "apache_beam.internal.code_object_pickler_test.ClassWithClassMethod" + ".class_method.__code__" ), ( ClassWithNestedFunction().process(), ( - "apache_beam.internal.code_object_pickler_test.ClassWithNestedFunction.process.__code__.co_consts" + "apache_beam.internal.code_object_pickler_test" + ".ClassWithNestedFunction.process.__code__.co_consts" "[nested_function]")), ( ClassWithLambda().process(), - "apache_beam.internal.code_object_pickler_test.ClassWithLambda.process.__code__.co_consts[]" + "apache_beam.internal.code_object_pickler_test.ClassWithLambda.process" + ".__code__.co_consts[]" ), ( ClassWithNestedClass.InnerClass().process, - "apache_beam.internal.code_object_pickler_test.ClassWithNestedClass.InnerClass.process.__code__" + "apache_beam.internal.code_object_pickler_test.ClassWithNestedClass" + ".InnerClass.process.__code__" ), ( ClassWithNestedLambda().process(), ( - "apache_beam.internal.code_object_pickler_test.ClassWithNestedLambda.process.__code__.co_consts" - "[get_lambda_from_dictionary].co_consts[, ('x',)]")), + "apache_beam.internal.code_object_pickler_test.ClassWithNestedLambda" + ".process.__code__.co_consts[get_lambda_from_dictionary].co_consts" + "[, ('x',)]")), ( ClassWithNestedLambda.process, - "apache_beam.internal.code_object_pickler_test.ClassWithNestedLambda.process.__code__" + "apache_beam.internal.code_object_pickler_test.ClassWithNestedLambda" + ".process.__code__" ), ] From c5a985a1636298c38cd2138c9f17de7b4bc77775 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Fri, 25 Jul 2025 14:43:13 -0400 Subject: [PATCH 074/142] Update code_object_pickler_test.py --- .../internal/code_object_pickler_test.py | 33 +++++++------------ 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index c9cf1fe428da..85376c07cf8f 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -106,13 +106,11 @@ def get_lambda_from_dictionary(): ( top_level_function, "apache_beam.internal.code_object_pickler_test.top_level_function" - ".__code__" - ), + ".__code__"), ( top_level_lambda, "apache_beam.internal.code_object_pickler_test.top_level_lambda" - ".__code__" - ), + ".__code__"), ( get_nested_function(), ( @@ -122,7 +120,8 @@ def get_lambda_from_dictionary(): get_lambda_from_dictionary(), ( "apache_beam.internal.code_object_pickler_test" - ".get_lambda_from_dictionary.__code__.co_consts[, ('x',)]")), + ".get_lambda_from_dictionary.__code__.co_consts[, ('x',)]") + ), ( function_with_lambda_default_argument(), ( @@ -131,28 +130,23 @@ def get_lambda_from_dictionary(): ( function_with_function_default_argument(), "apache_beam.internal.code_object_pickler_test.top_level_function" - ".__code__" - ), + ".__code__"), ( add_one, "apache_beam.internal.code_object_pickler_test.function_decorator" - ".__code__.co_consts[]" - ), + ".__code__.co_consts[]"), ( ClassWithFunction.process, "apache_beam.internal.code_object_pickler_test.ClassWithFunction.process" - ".__code__" - ), + ".__code__"), ( ClassWithStaticMethod.static_method, "apache_beam.internal.code_object_pickler_test.ClassWithStaticMethod" - ".static_method.__code__" - ), + ".static_method.__code__"), ( ClassWithClassMethod.class_method, "apache_beam.internal.code_object_pickler_test.ClassWithClassMethod" - ".class_method.__code__" - ), + ".class_method.__code__"), ( ClassWithNestedFunction().process(), ( @@ -162,13 +156,11 @@ def get_lambda_from_dictionary(): ( ClassWithLambda().process(), "apache_beam.internal.code_object_pickler_test.ClassWithLambda.process" - ".__code__.co_consts[]" - ), + ".__code__.co_consts[]"), ( ClassWithNestedClass.InnerClass().process, "apache_beam.internal.code_object_pickler_test.ClassWithNestedClass" - ".InnerClass.process.__code__" - ), + ".InnerClass.process.__code__"), ( ClassWithNestedLambda().process(), ( @@ -178,8 +170,7 @@ def get_lambda_from_dictionary(): ( ClassWithNestedLambda.process, "apache_beam.internal.code_object_pickler_test.ClassWithNestedLambda" - ".process.__code__" - ), + ".process.__code__"), ] From 60362bc527835afa6650d3accff0fd0d14827fec Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Fri, 25 Jul 2025 15:18:19 -0400 Subject: [PATCH 075/142] Update code_object_pickler_test.py --- sdks/python/apache_beam/internal/code_object_pickler_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index 85376c07cf8f..4209a79fbd04 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -105,7 +105,7 @@ def get_lambda_from_dictionary(): test_cases = [ ( top_level_function, - "apache_beam.internal.code_object_pickler_test.top_level_function" + "apache_beam.internal.code_object_pickler_test.top_level_function" ".__code__"), ( top_level_lambda, From 67087742c0be382b74540747a74461ea6b329f76 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Mon, 28 Jul 2025 13:28:09 -0400 Subject: [PATCH 076/142] Update code_object_pickler_test.py --- .../apache_beam/internal/code_object_pickler_test.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index 4209a79fbd04..98dc4e7f2c16 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -40,6 +40,11 @@ def get_lambda_from_dictionary(): return d["a"] +def get_lambda_from_dictionary_same_args(): + d = {"a": lambda x: 1, "b": lambda x: x + 1} + return d["a"] + + def function_with_lambda_default_argument(fn=lambda x: 1): return fn @@ -122,6 +127,13 @@ def get_lambda_from_dictionary(): "apache_beam.internal.code_object_pickler_test" ".get_lambda_from_dictionary.__code__.co_consts[, ('x',)]") ), + ( + get_lambda_from_dictionary_same_args(), + ( + "apache_beam.internal.code_object_pickler_test" + ".get_lambda_from_dictionary.__code__.co_consts[, ('x',), " + + hashlib.md5(.... .__code__).hexdigest() + "]") + ), ( function_with_lambda_default_argument(), ( From 6802a647cbab5e18bdea66d39c0f4f2877417ed7 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Mon, 28 Jul 2025 13:56:29 -0400 Subject: [PATCH 077/142] Update code_object_pickler_test.py --- sdks/python/apache_beam/internal/code_object_pickler_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index 98dc4e7f2c16..36343de974b4 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -132,7 +132,7 @@ def get_lambda_from_dictionary(): ( "apache_beam.internal.code_object_pickler_test" ".get_lambda_from_dictionary.__code__.co_consts[, ('x',), " + - hashlib.md5(.... .__code__).hexdigest() + "]") + hashlib.md5(get_lambda_from_dictionary.__code__).hexdigest() + "]") ), ( function_with_lambda_default_argument(), From 759205043529acaccfd39c9d443cc7340adb6171 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Mon, 28 Jul 2025 14:14:46 -0400 Subject: [PATCH 078/142] Update code_object_pickler_test.py --- sdks/python/apache_beam/internal/code_object_pickler_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index 36343de974b4..3379ea8ff991 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -15,6 +15,7 @@ # limitations under the License. # +import hashlib import unittest from apache_beam.internal import code_object_pickler From b442dece2fad96dd39e4a09bfd139fe1cc22e20a Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Mon, 28 Jul 2025 14:58:22 -0400 Subject: [PATCH 079/142] Update code_object_pickler_test.py --- sdks/python/apache_beam/internal/code_object_pickler_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index 3379ea8ff991..a23c0a4f21dc 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -133,7 +133,7 @@ def get_lambda_from_dictionary(): ( "apache_beam.internal.code_object_pickler_test" ".get_lambda_from_dictionary.__code__.co_consts[, ('x',), " + - hashlib.md5(get_lambda_from_dictionary.__code__).hexdigest() + "]") + hashlib.md5(get_lambda_from_dictionary.co_code).hexdigest() + "]") ), ( function_with_lambda_default_argument(), From 98df16351ce6acbbfe5c511c81e8247017b2b4cb Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Mon, 28 Jul 2025 15:18:02 -0400 Subject: [PATCH 080/142] Update code_object_pickler_test.py --- sdks/python/apache_beam/internal/code_object_pickler_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index a23c0a4f21dc..659319ca2447 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -133,7 +133,7 @@ def get_lambda_from_dictionary(): ( "apache_beam.internal.code_object_pickler_test" ".get_lambda_from_dictionary.__code__.co_consts[, ('x',), " + - hashlib.md5(get_lambda_from_dictionary.co_code).hexdigest() + "]") + hashlib.md5(get_lambda_from_dictionary.__code__.co_code).hexdigest() + "]") ), ( function_with_lambda_default_argument(), From 226885cc5bee2e311fd238b431e87f9e8c554989 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Mon, 28 Jul 2025 16:57:35 -0400 Subject: [PATCH 081/142] Update code_object_pickler_test.py --- .../apache_beam/internal/code_object_pickler_test.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index 659319ca2447..cefcf894facb 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -132,9 +132,10 @@ def get_lambda_from_dictionary(): get_lambda_from_dictionary_same_args(), ( "apache_beam.internal.code_object_pickler_test" - ".get_lambda_from_dictionary.__code__.co_consts[, ('x',), " + - hashlib.md5(get_lambda_from_dictionary.__code__.co_code).hexdigest() + "]") - ), + ".get_lambda_from_dictionary.__code__.co_consts[, ('x',), " + + hashlib.md5( + get_lambda_from_dictionary.__code__.co_code).hexdigest() + + "]")), ( function_with_lambda_default_argument(), ( From ba55678f49c9a0d53b6cd71faedbfffbc1908525 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 29 Jul 2025 09:56:08 -0400 Subject: [PATCH 082/142] Update code_object_pickler_test.py --- sdks/python/apache_beam/internal/code_object_pickler_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index cefcf894facb..25d829727812 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -132,9 +132,9 @@ def get_lambda_from_dictionary(): get_lambda_from_dictionary_same_args(), ( "apache_beam.internal.code_object_pickler_test" - ".get_lambda_from_dictionary.__code__.co_consts[, ('x',), " + ".get_lambda_from_dictionary_same_args.__code__.co_consts[, ('x',), " + hashlib.md5( - get_lambda_from_dictionary.__code__.co_code).hexdigest() + + get_lambda_from_dictionary_same_args.__code__.co_code).hexdigest() + "]")), ( function_with_lambda_default_argument(), From 4b7505f10b24d0242013498c06a19396a24bace2 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 29 Jul 2025 13:25:06 -0400 Subject: [PATCH 083/142] Update code_object_pickler_test.py --- sdks/python/apache_beam/internal/code_object_pickler_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index 25d829727812..d71cdacf3eb5 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -134,7 +134,7 @@ def get_lambda_from_dictionary(): "apache_beam.internal.code_object_pickler_test" ".get_lambda_from_dictionary_same_args.__code__.co_consts[, ('x',), " + hashlib.md5( - get_lambda_from_dictionary_same_args.__code__.co_code).hexdigest() + + get_lambda_from_dictionary_same_args().__code__.co_code).hexdigest() + "]")), ( function_with_lambda_default_argument(), @@ -188,7 +188,7 @@ def get_lambda_from_dictionary(): ] -class DillTest(unittest.TestCase): +class CodeObjectPicklerTest(unittest.TestCase): @parameterized.expand(test_cases) def test_get_code_path(self, callable, expected): actual = code_object_pickler._get_code_path(callable) From 4410b25e8347bc4f69a5342ac74a855a358333b8 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 29 Jul 2025 13:55:21 -0400 Subject: [PATCH 084/142] Update code_object_pickler_test.py --- sdks/python/apache_beam/internal/code_object_pickler_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index d71cdacf3eb5..3bc0f50cc6f2 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -134,8 +134,8 @@ def get_lambda_from_dictionary(): "apache_beam.internal.code_object_pickler_test" ".get_lambda_from_dictionary_same_args.__code__.co_consts[, ('x',), " + hashlib.md5( - get_lambda_from_dictionary_same_args().__code__.co_code).hexdigest() + - "]")), + get_lambda_from_dictionary_same_args().__code__.co_code + ).hexdigest() + "]")), ( function_with_lambda_default_argument(), ( From c275851ff5a920e50a69eb01bd50283867682c32 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 10:59:06 -0400 Subject: [PATCH 085/142] Update code_object_pickler_test.py --- .../internal/code_object_pickler_test.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index 3bc0f50cc6f2..c1a7e08c3ca5 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -132,7 +132,8 @@ def get_lambda_from_dictionary(): get_lambda_from_dictionary_same_args(), ( "apache_beam.internal.code_object_pickler_test" - ".get_lambda_from_dictionary_same_args.__code__.co_consts[, ('x',), " + ".get_lambda_from_dictionary_same_args.__code__.co_consts" + "[, ('x',), " + hashlib.md5( get_lambda_from_dictionary_same_args().__code__.co_code ).hexdigest() + "]")), @@ -151,8 +152,8 @@ def get_lambda_from_dictionary(): ".__code__.co_consts[]"), ( ClassWithFunction.process, - "apache_beam.internal.code_object_pickler_test.ClassWithFunction.process" - ".__code__"), + "apache_beam.internal.code_object_pickler_test.ClassWithFunction" + ".process.__code__"), ( ClassWithStaticMethod.static_method, "apache_beam.internal.code_object_pickler_test.ClassWithStaticMethod" @@ -178,9 +179,9 @@ def get_lambda_from_dictionary(): ( ClassWithNestedLambda().process(), ( - "apache_beam.internal.code_object_pickler_test.ClassWithNestedLambda" - ".process.__code__.co_consts[get_lambda_from_dictionary].co_consts" - "[, ('x',)]")), + "apache_beam.internal.code_object_pickler_test" + ".ClassWithNestedLambda.process.__code__.co_consts" + "[get_lambda_from_dictionary].co_consts""[, ('x',)]")), ( ClassWithNestedLambda.process, "apache_beam.internal.code_object_pickler_test.ClassWithNestedLambda" From cd97273a5c85fc2e6f8b6bcd166ef4231f05aa3f Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:00:28 -0400 Subject: [PATCH 086/142] Update pickle_code_path_test.py --- sdks/python/apache_beam/internal/pickle_code_path_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/python/apache_beam/internal/pickle_code_path_test.py b/sdks/python/apache_beam/internal/pickle_code_path_test.py index ec7ed0e87c28..50edef65f81b 100644 --- a/sdks/python/apache_beam/internal/pickle_code_path_test.py +++ b/sdks/python/apache_beam/internal/pickle_code_path_test.py @@ -19,6 +19,7 @@ import unittest +# pylint: disable=unused-import from apache_beam.internal import code_object_pickler from apache_beam.internal.test_cases import after_module_add_function from apache_beam.internal.test_cases import after_module_add_lambda_variable From 9ad075df69ac399dc49251bf023a37c6e70224dc Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:01:31 -0400 Subject: [PATCH 087/142] Update after_module_with_nested_function.py --- .../internal/test_cases/after_module_with_nested_function.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py index 10ba52c4d872..228fb8f57fac 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py @@ -19,10 +19,10 @@ def my_function(): - a = 1 + a = 1 # pylint: disable=unused-variable b = lambda: 2 - def nested_function(): + def nested_function(): # pylint: disable=unused-variable c = 3 return c From 51b1c13766cea904e4208e29743748dc6134abfd Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:02:48 -0400 Subject: [PATCH 088/142] Update after_module_with_classes.py --- .../test_cases/after_module_with_classes.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py index 59f02579b87c..db58590df367 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py @@ -20,9 +20,9 @@ class AddLocalVariable: def my_method(self): - a = 1 + a = 1 # pylint: disable=unused-variable b = lambda: 2 - new_local_variable = 3 + new_local_variable = 3 # pylint: disable=unused-variable return b @@ -34,9 +34,9 @@ def my_method(self): class AddLambdaVariable: def my_method(self): - a = 1 + a = 1 # pylint: disable=unused-variable b = lambda: 2 - c = lambda: 3 + c = lambda: 3 # pylint: disable=unused-variable return b @@ -48,21 +48,21 @@ def my_method(self): class ClassWithNestedFunction: def my_method(self): - def nested_function(): + def nested_function(): # pylint: disable=unused-variable c = 3 return c - a = 1 + a = 1 # pylint: disable=unused-variable b = lambda: 2 return b class ClassWithNestedFunction2: def my_method(self): - a = 1 + a = 1 # pylint: disable=unused-variable b = lambda: 2 - def nested_function(): + def nested_function(): # pylint: disable=unused-variable c = 3 return c @@ -71,19 +71,19 @@ def nested_function(): class ClassWithTwoMethods: def another_method(self): - a = 1 + a = 1 # pylint: disable=unused-variable b = lambda: 2 return b def my_method(self): - a = 1 + a = 1 # pylint: disable=unused-variable b = lambda: 2 return b class RemoveMethod: def my_method(self): - a = 1 + a = 1 # pylint: disable=unused-variable b = lambda: 2 return b From 13f0065e7296ac7e25761c876986209724454a42 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:03:12 -0400 Subject: [PATCH 089/142] Update after_module_with_single_class.py --- .../internal/test_cases/after_module_with_single_class.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_single_class.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_single_class.py index 8bffdf99f1ae..17da6b70e6e9 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_with_single_class.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_single_class.py @@ -20,12 +20,12 @@ class MyClass: def another_function(self): - a = 1 + a = 1 # pylint: disable=unused-variable b = lambda: 2 return b def my_function(): - a = 1 + a = 1 # pylint: disable=unused-variable b = lambda: 2 return b From 088be849501689865378c29a0fe8681556eb19a8 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:03:42 -0400 Subject: [PATCH 090/142] Update before_module_with_functions.py --- .../internal/test_cases/before_module_with_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/test_cases/before_module_with_functions.py b/sdks/python/apache_beam/internal/test_cases/before_module_with_functions.py index 67e2a7be28f4..dfd3ca3dd5cc 100644 --- a/sdks/python/apache_beam/internal/test_cases/before_module_with_functions.py +++ b/sdks/python/apache_beam/internal/test_cases/before_module_with_functions.py @@ -19,6 +19,6 @@ def my_function(): - a = 1 + a = 1 # pylint: disable=unused-variable b = lambda: 2 return b From c9547cbb1c607834ce168bd272477c444f71e8ea Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:04:36 -0400 Subject: [PATCH 091/142] Update after_module_add_lambda_variable.py --- .../internal/test_cases/after_module_add_lambda_variable.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_add_lambda_variable.py b/sdks/python/apache_beam/internal/test_cases/after_module_add_lambda_variable.py index 1b7366bfda0f..a87185fc3d27 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_add_lambda_variable.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_add_lambda_variable.py @@ -19,7 +19,7 @@ def my_function(): - a = 1 + a = 1 # pylint: disable=unused-variable b = lambda: 2 - new_lambda_variable = lambda: 4 + new_lambda_variable = lambda: 4 # pylint: disable=unused-variable return b From 9f7300869efddf4a66cabf2cdd808e42a1bc30ea Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:04:52 -0400 Subject: [PATCH 092/142] Update after_module_with_global_variable.py --- .../internal/test_cases/after_module_with_global_variable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_global_variable.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_global_variable.py index 670e4108f737..3d6ac93c96e1 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_with_global_variable.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_global_variable.py @@ -21,6 +21,6 @@ def my_function(): - a = 1 + a = 1 # pylint: disable=unused-variable b = lambda: 2 return b From 180282dbeb35ac5f20edbd9d45e75f694b6ba3d3 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:05:15 -0400 Subject: [PATCH 093/142] Update after_module_with_nested_function_2.py --- .../test_cases/after_module_with_nested_function_2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function_2.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function_2.py index 30744f0bbf89..c75eb1d6c77b 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function_2.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function_2.py @@ -19,10 +19,10 @@ def my_function(): - def nested_function(): + def nested_function(): # pylint: disable=unused-variable c = 3 return c - a = 1 + a = 1 # pylint: disable=unused-variable b = lambda: 2 return b From 140bba9c7f92385a6eedc753ce0c41d7de68e474 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:05:41 -0400 Subject: [PATCH 094/142] Update after_module_add_function.py --- .../internal/test_cases/after_module_add_function.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py b/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py index 6c9d83567da5..d7d68fa57e7f 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py @@ -19,12 +19,12 @@ def another_function(): - a = 1 + a = 1 # pylint: disable=unused-variable b = lambda: 2 return b def my_function(): - a = 1 + a = 1 # pylint: disable=unused-variable b = lambda: 2 return b From 2fb0791241305db3f6025dd0d338246179983d4e Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:06:02 -0400 Subject: [PATCH 095/142] Update before_module_with_lambdas.py --- .../internal/test_cases/before_module_with_lambdas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/test_cases/before_module_with_lambdas.py b/sdks/python/apache_beam/internal/test_cases/before_module_with_lambdas.py index 88b570f18dc0..0adb3cb41cda 100644 --- a/sdks/python/apache_beam/internal/test_cases/before_module_with_lambdas.py +++ b/sdks/python/apache_beam/internal/test_cases/before_module_with_lambdas.py @@ -19,6 +19,6 @@ def my_function(): - a = lambda: 1 + a = lambda: 1 # pylint: disable=unused-variable b = lambda: 2 return b From 51c276632ea581898b26cbdc7559f6f192ce0b57 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:06:58 -0400 Subject: [PATCH 096/142] Update before_module_with_classes.py --- .../test_cases/before_module_with_classes.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/sdks/python/apache_beam/internal/test_cases/before_module_with_classes.py b/sdks/python/apache_beam/internal/test_cases/before_module_with_classes.py index c9917ee1bede..421635c32ce9 100644 --- a/sdks/python/apache_beam/internal/test_cases/before_module_with_classes.py +++ b/sdks/python/apache_beam/internal/test_cases/before_module_with_classes.py @@ -20,60 +20,60 @@ class AddLocalVariable: def my_method(self): - a = 1 + a = 1 # pylint: disable=unused-variable b = lambda: 2 return b class RemoveLocalVariable: def my_method(self): - a = 1 + a = 1 # pylint: disable=unused-variable b = lambda: 2 return b class AddLambdaVariable: def my_method(self): - a = 1 + a = 1 # pylint: disable=unused-variable b = lambda: 2 return b class RemoveLambdaVariable: def my_method(self): - a = lambda: 1 + a = lambda: 1 # pylint: disable=unused-variable b = lambda: 2 return b class ClassWithNestedFunction: def my_method(self): - a = 1 + a = 1 # pylint: disable=unused-variable b = lambda: 2 return b class ClassWithNestedFunction2: def my_method(self): - a = 1 + a = 1 # pylint: disable=unused-variable b = lambda: 2 return b class ClassWithTwoMethods: def my_method(self): - a = 1 + a = 1 # pylint: disable=unused-variable b = lambda: 2 return b class RemoveMethod: def another_method(self): - a = 1 + a = 1 # pylint: disable=unused-variable b = lambda: 2 return b def my_method(self): - a = 1 + a = 1 # pylint: disable=unused-variable b = lambda: 2 return b From e126130f10dfc03981f979dbde2ab90f41576e3f Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:07:28 -0400 Subject: [PATCH 097/142] Update after_module_add_variable.py --- .../internal/test_cases/after_module_add_variable.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_add_variable.py b/sdks/python/apache_beam/internal/test_cases/after_module_add_variable.py index bf70147a9027..78b5b1fb1d8f 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_add_variable.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_add_variable.py @@ -19,7 +19,7 @@ def my_function(): - a = 1 + a = 1 # pylint: disable=unused-variable b = lambda: 2 - new_local_variable = 3 + new_local_variable = 3 # pylint: disable=unused-variable return b From 59659bb6e48e46623e9cd671364e9f6b0925cfad Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:39:04 -0400 Subject: [PATCH 098/142] Update code_object_pickler.py --- .../internal/code_object_pickler.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/sdks/python/apache_beam/internal/code_object_pickler.py b/sdks/python/apache_beam/internal/code_object_pickler.py index 25cf92dc348f..7daa572bbdb4 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler.py +++ b/sdks/python/apache_beam/internal/code_object_pickler.py @@ -15,6 +15,36 @@ # limitations under the License. # +"""Customizations to how Python code objects are pickled. + +This module provides functions for pickling code objects, especially lambdas, +in a consistent way. It addresses issues with non-deterministic pickling by +creating a unique identifier that is invariant to small changes in the source +code. + +The code object identifiers consists of a sequence of the following parts +separated by periods: +- Module names - The name of the module the code object is in +- Class names - The name of a class containing the code object. There can be + multiple of these in the same identifier in the case of nested + classes. +- Function names - The name of the function containing the code object. + There can be multiple of these in the case of nested functions. +- __code__ - Attribute indicating that we are entering the code object of a + function/method. +- __co_consts__[] - The name of the local variable containing the + code object. In the case of lambdas, the name is created by using the + signature of the lambda and hashing the bytecode, as shown below. + +Examples: +- __main__.top_level_function.__code__ +- __main__.ClassWithNestedFunction.process.__code__.co_consts[nested_function] +- __main__.ClassWithNestedLambda.process.__code__.co_consts[ + get_lambda_from_dictionary].co_consts[, ('x',)] +- __main__.ClassWithNestedLambda.process.__code__.co_consts[ + , ('x',), 1234567890] +""" + import collections import hashlib import inspect From 3fde5a02c8c7698ffa3aa3320988dbf6234f482e Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:47:16 -0400 Subject: [PATCH 099/142] Update pickle_code_path_test.py --- .../internal/pickle_code_path_test.py | 78 +++++++++---------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/sdks/python/apache_beam/internal/pickle_code_path_test.py b/sdks/python/apache_beam/internal/pickle_code_path_test.py index 50edef65f81b..c3c7e87b35cc 100644 --- a/sdks/python/apache_beam/internal/pickle_code_path_test.py +++ b/sdks/python/apache_beam/internal/pickle_code_path_test.py @@ -38,38 +38,38 @@ class CodePathTest(unittest.TestCase): - def test_get_code_from_stable_reference_empty_path(self): + def test_get_code_from_stable_reference_empty_path_raises_exception(self): with self.assertRaisesRegex(ValueError, "Path must not be empty"): code_object_pickler._get_code_from_stable_reference("") - def test_get_code_from_stable_reference_invalid_default_index(self): + def test_get_code_from_stable_reference_invalid_default_index_raises_exception(self): with self.assertRaisesRegex(ValueError, "out of bounds"): code_object_pickler._get_code_from_stable_reference( "apache_beam.internal.test_cases.module_with_default_argument." "function_with_lambda_default_argument.__defaults__[1]") - def test_get_code_from_stable_reference_invalid_single_name_path(self): + def test_get_code_from_stable_reference_invalid_single_name_path_raises_exception(self): with self.assertRaisesRegex(AttributeError, "Could not find code object with path"): code_object_pickler._get_code_from_stable_reference( "apache_beam.internal.test_cases.before_module_with_lambdas." "my_function.__code__.co_consts[something]") - def test_get_code_from_stable_reference_invalid_lambda_with_args_path(self): + def test_get_code_from_stable_reference_invalid_lambda_with_args_path_raises_exception(self): with self.assertRaisesRegex(AttributeError, "Could not find code object with path"): code_object_pickler._get_code_from_stable_reference( "apache_beam.internal.test_cases.before_module_with_lambdas." "my_function.__code__.co_consts[, ('x',)]") - def test_get_code_from_stable_reference_invalid_lambda_with_hash_path(self): + def test_get_code_from_stable_reference_invalid_lambda_with_hash_path_raises_exception(self): with self.assertRaisesRegex(AttributeError, "Could not find code object with path"): code_object_pickler._get_code_from_stable_reference( "apache_beam.internal.test_cases.before_module_with_lambdas." "my_function.__code__.co_consts[, ('',), 1234567890]") - def test_get_code_add_local_variable_in_class(self): + def test_get_code_add_local_variable_in_class_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( @@ -80,7 +80,7 @@ def test_get_code_add_local_variable_in_class(self): after_module_with_classes.AddLocalVariable.my_method(self).__code__, ) - def test_get_code_remove_local_variable_in_class(self): + def test_get_code_remove_local_variable_in_class_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( @@ -91,7 +91,7 @@ def test_get_code_remove_local_variable_in_class(self): after_module_with_classes.RemoveLocalVariable.my_method(self).__code__, ) - def test_get_code_add_lambda_variable_in_class(self): + def test_get_code_add_lambda_variable_in_class_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( @@ -102,7 +102,7 @@ def test_get_code_add_lambda_variable_in_class(self): after_module_with_classes.AddLambdaVariable.my_method(self).__code__, ) - def test_get_code_remove_lambda_variable_in_class(self): + def test_get_code_remove_lambda_variable_in_class_objects_match(self): with self.assertRaisesRegex(AttributeError, "object has no attribute"): code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( @@ -111,7 +111,7 @@ def test_get_code_remove_lambda_variable_in_class(self): "before_module_with_classes", "after_module_with_classes")) - def test_get_code_nested_function_in_class(self): + def test_get_code_nested_function_in_class_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( @@ -123,7 +123,7 @@ def test_get_code_nested_function_in_class(self): self).__code__, ) - def test_get_code_nested_function_2_in_class(self): + def test_get_code_nested_function_2_in_class_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( @@ -135,7 +135,7 @@ def test_get_code_nested_function_2_in_class(self): self).__code__, ) - def test_get_code_add_new_function_in_class(self): + def test_get_code_add_new_function_in_class_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( @@ -146,7 +146,7 @@ def test_get_code_add_new_function_in_class(self): after_module_with_classes.ClassWithTwoMethods.my_method(self).__code__, ) - def test_get_code_remove_method_in_class(self): + def test_get_code_remove_method_in_class_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( @@ -157,7 +157,7 @@ def test_get_code_remove_method_in_class(self): after_module_with_classes.RemoveMethod.my_method(self).__code__, ) - def test_get_code_add_global_variable(self): + def test_get_code_add_global_variable_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( @@ -168,7 +168,7 @@ def test_get_code_add_global_variable(self): after_module_with_global_variable.my_function().__code__, ) - def test_get_code_add_top_level_function(self): + def test_get_code_add_top_level_function_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( @@ -178,7 +178,7 @@ def test_get_code_add_top_level_function(self): after_module_add_function.my_function().__code__, ) - def test_get_code_add_local_variable_in_function(self): + def test_get_code_add_local_variable_in_function_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( @@ -188,7 +188,7 @@ def test_get_code_add_local_variable_in_function(self): after_module_add_variable.my_function().__code__, ) - def test_get_code_remove_local_variable_in_function(self): + def test_get_code_remove_local_variable_in_function_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( @@ -198,7 +198,7 @@ def test_get_code_remove_local_variable_in_function(self): after_module_remove_variable.my_function().__code__, ) - def test_get_code_nested_function_in_function(self): + def test_get_code_nested_function_in_function_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( @@ -208,7 +208,7 @@ def test_get_code_nested_function_in_function(self): after_module_with_nested_function.my_function().__code__, ) - def test_get_code_nested_function_2_in_function(self): + def test_get_code_nested_function_2_in_function_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( @@ -218,7 +218,7 @@ def test_get_code_nested_function_2_in_function(self): after_module_with_nested_function_2.my_function().__code__, ) - def test_get_code_add_class(self): + def test_get_code_add_class_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( @@ -228,7 +228,7 @@ def test_get_code_add_class(self): after_module_with_single_class.my_function().__code__, ) - def test_get_code_add_lambda_variable_in_function(self): + def test_get_code_add_lambda_variable_in_function_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( @@ -238,7 +238,7 @@ def test_get_code_add_lambda_variable_in_function(self): after_module_add_lambda_variable.my_function().__code__, ) - def test_get_code_remove_lambda_variable_in_function(self): + def test_get_code_remove_lambda_variable_in_function_raises_exception(self): with self.assertRaisesRegex(AttributeError, "object has no attribute"): code_object_pickler._get_code_from_stable_reference( code_object_pickler._get_code_path( @@ -246,7 +246,7 @@ def test_get_code_remove_lambda_variable_in_function(self): "before_module_with_lambdas", "after_module_remove_lambda_variable")) - def test_identifiers_new_local_variable_in_class(self): + def test_identifiers_new_local_variable_in_class_paths_match(self): self.assertEqual( code_object_pickler._get_code_path( before_module_with_classes.AddLocalVariable.my_method( @@ -256,7 +256,7 @@ def test_identifiers_new_local_variable_in_class(self): "after_module_with_classes", "module_name"), ) - def test_identifiers_remove_local_variable_in_class(self): + def test_identifiers_remove_local_variable_in_class_paths_match(self): self.assertEqual( code_object_pickler._get_code_path( before_module_with_classes.RemoveLocalVariable.my_method( @@ -266,7 +266,7 @@ def test_identifiers_remove_local_variable_in_class(self): self)).replace("after_module_with_classes", "module_name"), ) - def test_identifiers_add_lambda_variable_in_class(self): + def test_identifiers_add_lambda_variable_in_class_paths_match(self): self.assertNotEqual( code_object_pickler._get_code_path( before_module_with_classes.AddLambdaVariable.my_method( @@ -276,7 +276,7 @@ def test_identifiers_add_lambda_variable_in_class(self): self)).replace("after_module_with_classes", "module_name"), ) - def test_identifiers_remove_lambda_variable_in_class(self): + def test_identifiers_remove_lambda_variable_in_class_paths_match(self): self.assertNotEqual( code_object_pickler._get_code_path( before_module_with_classes.RemoveLambdaVariable.my_method( @@ -286,7 +286,7 @@ def test_identifiers_remove_lambda_variable_in_class(self): self)).replace("after_module_with_classes", "module_name"), ) - def test_identifiers_nested_function_in_class(self): + def test_identifiers_nested_function_in_class_paths_match(self): self.assertEqual( code_object_pickler._get_code_path( before_module_with_classes.ClassWithNestedFunction.my_method( @@ -296,7 +296,7 @@ def test_identifiers_nested_function_in_class(self): self)).replace("after_module_with_classes", "module_name"), ) - def test_identifiers_nested_function_2_in_class(self): + def test_identifiers_nested_function_2_in_class_paths_match(self): self.assertEqual( code_object_pickler._get_code_path( before_module_with_classes.ClassWithNestedFunction2.my_method( @@ -306,7 +306,7 @@ def test_identifiers_nested_function_2_in_class(self): self)).replace("after_module_with_classes", "module_name"), ) - def test_identifiers_add_new_function_in_class(self): + def test_identifiers_add_new_function_in_class_paths_match(self): self.assertEqual( code_object_pickler._get_code_path( before_module_with_classes.ClassWithTwoMethods.my_method( @@ -316,7 +316,7 @@ def test_identifiers_add_new_function_in_class(self): self)).replace("after_module_with_classes", "module_name"), ) - def test_identifiers_remove_function_in_class(self): + def test_identifiers_remove_function_in_class_paths_match(self): self.assertEqual( code_object_pickler._get_code_path( before_module_with_classes.RemoveMethod.my_method(self)).replace( @@ -326,7 +326,7 @@ def test_identifiers_remove_function_in_class(self): "after_module_with_classes", "module_name"), ) - def test_identifiers_add_global_variable(self): + def test_identifiers_add_global_variable_paths_match(self): self.assertEqual( code_object_pickler._get_code_path( before_module_with_functions.my_function()).replace( @@ -336,7 +336,7 @@ def test_identifiers_add_global_variable(self): "after_module_with_global_variable", "module_name"), ) - def test_identifiers_add_top_level_function(self): + def test_identifiers_add_top_level_function_paths_match(self): self.assertEqual( code_object_pickler._get_code_path( before_module_with_functions.my_function()).replace( @@ -346,7 +346,7 @@ def test_identifiers_add_top_level_function(self): "after_module_add_function", "module_name"), ) - def test_identifiers_add_local_variable_in_function(self): + def test_identifiers_add_local_variable_in_function_paths_match(self): self.assertEqual( code_object_pickler._get_code_path( before_module_with_functions.my_function()).replace( @@ -356,7 +356,7 @@ def test_identifiers_add_local_variable_in_function(self): "after_module_add_variable", "module_name"), ) - def test_identifiers_remove_local_variable_in_function(self): + def test_identifiers_remove_local_variable_in_function_paths_match(self): self.assertEqual( code_object_pickler._get_code_path( before_module_with_functions.my_function()).replace( @@ -366,7 +366,7 @@ def test_identifiers_remove_local_variable_in_function(self): "after_module_remove_variable", "module_name"), ) - def test_identifiers_nested_function_in_function(self): + def test_identifiers_nested_function_in_function_paths_match(self): self.assertEqual( code_object_pickler._get_code_path( before_module_with_functions.my_function()).replace( @@ -376,7 +376,7 @@ def test_identifiers_nested_function_in_function(self): "after_module_with_nested_function", "module_name"), ) - def test_identifiers_nested_function_2_in_function(self): + def test_identifiers_nested_function_2_in_function_paths_match(self): self.assertEqual( code_object_pickler._get_code_path( before_module_with_functions.my_function()).replace( @@ -386,7 +386,7 @@ def test_identifiers_nested_function_2_in_function(self): "after_module_with_nested_function_2", "module_name"), ) - def test_identifiers_add_class(self): + def test_identifiers_add_class_paths_match(self): self.assertEqual( code_object_pickler._get_code_path( before_module_with_functions.my_function()).replace( @@ -396,7 +396,7 @@ def test_identifiers_add_class(self): "after_module_with_single_class", "module_name"), ) - def test_identifiers_add_lambda_variable_in_function(self): + def test_identifiers_add_lambda_variable_in_function_no_match(self): self.assertNotEqual( code_object_pickler._get_code_path( before_module_with_functions.my_function()).replace( @@ -406,7 +406,7 @@ def test_identifiers_add_lambda_variable_in_function(self): "after_module_add_lambda_variable", "module_name"), ) - def test_identifiers_remove_lambda_variable_in_function(self): + def test_identifiers_remove_lambda_variable_in_function_no_match(self): self.assertNotEqual( code_object_pickler._get_code_path( before_module_with_lambdas.my_function()).replace( From 6ec1d10839c159377dec492c0b5ee9b2fa531a7e Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:49:26 -0400 Subject: [PATCH 100/142] Update after_module_add_function.py --- .../internal/test_cases/after_module_add_function.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py b/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py index d7d68fa57e7f..06dbd3bff973 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py @@ -15,8 +15,10 @@ # limitations under the License. # -"""Module for testing code path generation with functions.""" - +"""Module for testing code path generation with functions. +Counterpart to before_module_with_functions and is used to test that +code paths don't change is a function is added to the code. +""" def another_function(): a = 1 # pylint: disable=unused-variable From f0d693cd30f9494fd913bc8f7b02d0c1be5146b7 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:52:30 -0400 Subject: [PATCH 101/142] Update after_module_add_lambda_variable.py --- .../internal/test_cases/after_module_add_lambda_variable.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_add_lambda_variable.py b/sdks/python/apache_beam/internal/test_cases/after_module_add_lambda_variable.py index a87185fc3d27..7e1ddc48790f 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_add_lambda_variable.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_add_lambda_variable.py @@ -15,7 +15,10 @@ # limitations under the License. # -"""Module for testing code path generation with functions.""" +"""Module for testing code path generation with functions. +Counterpart to before_module_with_functions and is used as a test case +for adding a lambda variable. +""" def my_function(): From 969ec6a27b4098687f45b3c8fdae518e9badac11 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:52:48 -0400 Subject: [PATCH 102/142] Update after_module_add_function.py --- .../internal/test_cases/after_module_add_function.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py b/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py index 06dbd3bff973..93b489bd3f63 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py @@ -16,8 +16,8 @@ # """Module for testing code path generation with functions. -Counterpart to before_module_with_functions and is used to test that -code paths don't change is a function is added to the code. +Counterpart to before_module_with_functions and is used as a test case +for adding a lambda function. """ def another_function(): From 6fc392074247e83bbbbc91ad4097ffefed683e2b Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:53:05 -0400 Subject: [PATCH 103/142] Update after_module_add_function.py --- .../internal/test_cases/after_module_add_function.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py b/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py index 93b489bd3f63..9529d17621e8 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py @@ -17,7 +17,7 @@ """Module for testing code path generation with functions. Counterpart to before_module_with_functions and is used as a test case -for adding a lambda function. +for adding a function. """ def another_function(): From e3692a5b2ad93df2ad0ff72525c5c55e1d95114a Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:53:34 -0400 Subject: [PATCH 104/142] Update after_module_add_variable.py --- .../internal/test_cases/after_module_add_variable.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_add_variable.py b/sdks/python/apache_beam/internal/test_cases/after_module_add_variable.py index 78b5b1fb1d8f..a808e7ca4e9b 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_add_variable.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_add_variable.py @@ -15,7 +15,10 @@ # limitations under the License. # -"""Module for testing code path generation with functions.""" +"""Module for testing code path generation with functions. +Counterpart to before_module_with_functions and is used as a test case +for adding a variable. +""" def my_function(): From 47de4931e6577740252836edce06df059f33b460 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:54:10 -0400 Subject: [PATCH 105/142] Update after_module_remove_lambda_variable.py --- .../test_cases/after_module_remove_lambda_variable.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_remove_lambda_variable.py b/sdks/python/apache_beam/internal/test_cases/after_module_remove_lambda_variable.py index 436f42877cf7..326dedb93f0d 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_remove_lambda_variable.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_remove_lambda_variable.py @@ -15,7 +15,10 @@ # limitations under the License. # -"""Module for testing code path generation with functions.""" +"""Module for testing code path generation with functions. +Counterpart to before_module_with_functions and is used as a test case +for removing a lambda variable. +""" def my_function(): From 2580238a6db514d661d226c1d9877988038c0529 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:54:30 -0400 Subject: [PATCH 106/142] Update after_module_remove_variable.py --- .../internal/test_cases/after_module_remove_variable.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_remove_variable.py b/sdks/python/apache_beam/internal/test_cases/after_module_remove_variable.py index 436f42877cf7..072b93d48447 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_remove_variable.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_remove_variable.py @@ -15,7 +15,10 @@ # limitations under the License. # -"""Module for testing code path generation with functions.""" +"""Module for testing code path generation with functions. +Counterpart to before_module_with_functions and is used as a test case +for removing a variable. +""" def my_function(): From f36c2bedd90e8a6afebc54406549165f17546d0c Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:54:57 -0400 Subject: [PATCH 107/142] Update after_module_with_classes.py --- .../internal/test_cases/after_module_with_classes.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py index db58590df367..891e493a011b 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py @@ -15,7 +15,10 @@ # limitations under the License. # -"""Module for testing code path generation with classes.""" +"""Module for testing code path generation with functions. +Counterpart to before_module_with_classes and is used as a test case +for various code changes. +""" class AddLocalVariable: From 6b63a36684b590aa31dd9be8e93a4c7e7590fbbe Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:55:20 -0400 Subject: [PATCH 108/142] Update after_module_with_global_variable.py --- .../internal/test_cases/after_module_with_global_variable.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_global_variable.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_global_variable.py index 3d6ac93c96e1..c4d20f27c837 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_with_global_variable.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_global_variable.py @@ -15,7 +15,10 @@ # limitations under the License. # -"""Module for testing code path generation with a global variable.""" +"""Module for testing code path generation with functions. +Counterpart to before_module_with_functions and is used as a test case +for adding a global variable. +""" GLOBAL_VARIABLE = lambda: 3 From d3898b3c11b211751c83f7e7d32a2915d6d3cc74 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:55:48 -0400 Subject: [PATCH 109/142] Update after_module_with_nested_function.py --- .../internal/test_cases/after_module_with_nested_function.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py index 228fb8f57fac..191e2e92065a 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py @@ -15,7 +15,10 @@ # limitations under the License. # -"""Module for testing code path generation with functions.""" +"""Module for testing code path generation with functions. +Counterpart to before_module_with_functions and is used as a test case +for adding a nested function. +""" def my_function(): From 152c3d59e31b090bb363c71dc3f2a56826ae3133 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:56:09 -0400 Subject: [PATCH 110/142] Update after_module_with_nested_function_2.py --- .../test_cases/after_module_with_nested_function_2.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function_2.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function_2.py index c75eb1d6c77b..358bf12b3d05 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function_2.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function_2.py @@ -15,7 +15,10 @@ # limitations under the License. # -"""Module for testing code path generation with functions.""" +"""Module for testing code path generation with functions. +Counterpart to before_module_with_functions and is used as a test case +for adding a nested function. +""" def my_function(): From f7c18b1ed2608baf347ff69819f4ac761d4d7e16 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:56:32 -0400 Subject: [PATCH 111/142] Update after_module_with_single_class.py --- .../internal/test_cases/after_module_with_single_class.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_single_class.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_single_class.py index 17da6b70e6e9..0a4a7f73974c 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_with_single_class.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_single_class.py @@ -15,7 +15,10 @@ # limitations under the License. # -"""Module for testing code path generation with a single class.""" +"""Module for testing code path generation with functions. +Counterpart to before_module_with_functions and is used as a test case +for adding a class. +""" class MyClass: From b829f5846cff5b30625f1b9824cd4a62104d1581 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:57:15 -0400 Subject: [PATCH 112/142] Update before_module_with_classes.py --- .../internal/test_cases/before_module_with_classes.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/test_cases/before_module_with_classes.py b/sdks/python/apache_beam/internal/test_cases/before_module_with_classes.py index 421635c32ce9..36326844520a 100644 --- a/sdks/python/apache_beam/internal/test_cases/before_module_with_classes.py +++ b/sdks/python/apache_beam/internal/test_cases/before_module_with_classes.py @@ -15,7 +15,10 @@ # limitations under the License. # -"""Module for testing code path generation with classes.""" +"""Module for testing code path generation with functions. +Counterpart to after_module_with_classes and is used as a test case +for various code changes. +""" class AddLocalVariable: From e0878ae5d6e86139caf1edc5585051f4ffa9f6ec Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:57:44 -0400 Subject: [PATCH 113/142] Update before_module_with_functions.py --- .../internal/test_cases/before_module_with_functions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/test_cases/before_module_with_functions.py b/sdks/python/apache_beam/internal/test_cases/before_module_with_functions.py index dfd3ca3dd5cc..9dd7ebf90881 100644 --- a/sdks/python/apache_beam/internal/test_cases/before_module_with_functions.py +++ b/sdks/python/apache_beam/internal/test_cases/before_module_with_functions.py @@ -15,7 +15,10 @@ # limitations under the License. # -"""Module for testing code path generation with functions.""" +"""Module for testing code path generation with functions. +Counterpart to after_module_with_functions and is used as a test case +for various code changes. +""" def my_function(): From c1374ec627d13e0e688facbe1e78ca3278a0c531 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 11:59:06 -0400 Subject: [PATCH 114/142] Update before_module_with_lambdas.py --- .../internal/test_cases/before_module_with_lambdas.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/test_cases/before_module_with_lambdas.py b/sdks/python/apache_beam/internal/test_cases/before_module_with_lambdas.py index 0adb3cb41cda..98f9c29ceb52 100644 --- a/sdks/python/apache_beam/internal/test_cases/before_module_with_lambdas.py +++ b/sdks/python/apache_beam/internal/test_cases/before_module_with_lambdas.py @@ -15,7 +15,9 @@ # limitations under the License. # -"""Module for testing code path generation with functions.""" +"""Module for testing code path generation with functions. +Used as a test case for various code changes. +""" def my_function(): From 0435f17fc9fd2364ae2f0632904821aeb0264fbd Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 12:00:01 -0400 Subject: [PATCH 115/142] Update module_with_default_argument.py --- .../internal/test_cases/module_with_default_argument.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/test_cases/module_with_default_argument.py b/sdks/python/apache_beam/internal/test_cases/module_with_default_argument.py index ae3366ae752a..a1758a8be740 100644 --- a/sdks/python/apache_beam/internal/test_cases/module_with_default_argument.py +++ b/sdks/python/apache_beam/internal/test_cases/module_with_default_argument.py @@ -15,7 +15,9 @@ # limitations under the License. # -"""Module for testing code path generation with functions.""" +"""Module for testing code path generation with functions. +Used as a test case for default arguments. +""" def function_with_lambda_default_argument(fn=lambda x: 1): From d587a4600e6c65fcae5bc9c92fd109cf0f02f20a Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 12:00:22 -0400 Subject: [PATCH 116/142] Update after_module_with_classes.py --- .../internal/test_cases/after_module_with_classes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py b/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py index 891e493a011b..56e57f8cb87b 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py @@ -15,7 +15,7 @@ # limitations under the License. # -"""Module for testing code path generation with functions. +"""Module for testing code path generation with classes. Counterpart to before_module_with_classes and is used as a test case for various code changes. """ From 128fbe258295169949047b4ec7c1fc3fbbffebcb Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 12:00:33 -0400 Subject: [PATCH 117/142] Update before_module_with_classes.py --- .../internal/test_cases/before_module_with_classes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/test_cases/before_module_with_classes.py b/sdks/python/apache_beam/internal/test_cases/before_module_with_classes.py index 36326844520a..e59deca69f0b 100644 --- a/sdks/python/apache_beam/internal/test_cases/before_module_with_classes.py +++ b/sdks/python/apache_beam/internal/test_cases/before_module_with_classes.py @@ -15,7 +15,7 @@ # limitations under the License. # -"""Module for testing code path generation with functions. +"""Module for testing code path generation with classes. Counterpart to after_module_with_classes and is used as a test case for various code changes. """ From 6bf9ec6c73fd2ce4e7d730205df7a933d5c791fb Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 13:02:24 -0400 Subject: [PATCH 118/142] Update code_object_pickler_test.py --- .../apache_beam/internal/code_object_pickler_test.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index c1a7e08c3ca5..4d18cf940ef5 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -133,10 +133,9 @@ def get_lambda_from_dictionary(): ( "apache_beam.internal.code_object_pickler_test" ".get_lambda_from_dictionary_same_args.__code__.co_consts" - "[, ('x',), " - + hashlib.md5( - get_lambda_from_dictionary_same_args().__code__.co_code - ).hexdigest() + "]")), + "[, ('x',), " + hashlib.md5( + get_lambda_from_dictionary_same_args().__code__.co_code). + hexdigest() + "]")), ( function_with_lambda_default_argument(), ( @@ -181,7 +180,7 @@ def get_lambda_from_dictionary(): ( "apache_beam.internal.code_object_pickler_test" ".ClassWithNestedLambda.process.__code__.co_consts" - "[get_lambda_from_dictionary].co_consts""[, ('x',)]")), + "[get_lambda_from_dictionary].co_consts[, ('x',)]")), ( ClassWithNestedLambda.process, "apache_beam.internal.code_object_pickler_test.ClassWithNestedLambda" From ffec1ef552149193847fb646628506d075fe4ffa Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 13:03:06 -0400 Subject: [PATCH 119/142] Update after_module_add_function.py --- .../apache_beam/internal/test_cases/after_module_add_function.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py b/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py index 9529d17621e8..063125a2500d 100644 --- a/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py +++ b/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py @@ -20,6 +20,7 @@ for adding a function. """ + def another_function(): a = 1 # pylint: disable=unused-variable b = lambda: 2 From d8005cac9f37fbc409a07a4f910e02b4fdd37886 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 13:06:33 -0400 Subject: [PATCH 120/142] Update pickle_code_path_test.py --- sdks/python/apache_beam/internal/pickle_code_path_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdks/python/apache_beam/internal/pickle_code_path_test.py b/sdks/python/apache_beam/internal/pickle_code_path_test.py index c3c7e87b35cc..8953ddf2e374 100644 --- a/sdks/python/apache_beam/internal/pickle_code_path_test.py +++ b/sdks/python/apache_beam/internal/pickle_code_path_test.py @@ -42,27 +42,27 @@ def test_get_code_from_stable_reference_empty_path_raises_exception(self): with self.assertRaisesRegex(ValueError, "Path must not be empty"): code_object_pickler._get_code_from_stable_reference("") - def test_get_code_from_stable_reference_invalid_default_index_raises_exception(self): + def test_get_code_from_stable_reference_invalid_default_index_raises_exception(self): # pylint: disable=line-too-long with self.assertRaisesRegex(ValueError, "out of bounds"): code_object_pickler._get_code_from_stable_reference( "apache_beam.internal.test_cases.module_with_default_argument." "function_with_lambda_default_argument.__defaults__[1]") - def test_get_code_from_stable_reference_invalid_single_name_path_raises_exception(self): + def test_get_code_from_stable_reference_invalid_single_name_path_raises_exception(self): # pylint: disable=line-too-long with self.assertRaisesRegex(AttributeError, "Could not find code object with path"): code_object_pickler._get_code_from_stable_reference( "apache_beam.internal.test_cases.before_module_with_lambdas." "my_function.__code__.co_consts[something]") - def test_get_code_from_stable_reference_invalid_lambda_with_args_path_raises_exception(self): + def test_get_code_from_stable_reference_invalid_lambda_with_args_path_raises_exception(self): # pylint: disable=line-too-long with self.assertRaisesRegex(AttributeError, "Could not find code object with path"): code_object_pickler._get_code_from_stable_reference( "apache_beam.internal.test_cases.before_module_with_lambdas." "my_function.__code__.co_consts[, ('x',)]") - def test_get_code_from_stable_reference_invalid_lambda_with_hash_path_raises_exception(self): + def test_get_code_from_stable_reference_invalid_lambda_with_hash_path_raises_exception(self): # pylint: disable=line-too-long with self.assertRaisesRegex(AttributeError, "Could not find code object with path"): code_object_pickler._get_code_from_stable_reference( From bfec969007e95ad6701bf2cab39ad747c21bf95f Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 5 Aug 2025 13:30:29 -0400 Subject: [PATCH 121/142] Update code_object_pickler_test.py --- sdks/python/apache_beam/internal/code_object_pickler_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index 4d18cf940ef5..6332ad97f4fc 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -18,9 +18,10 @@ import hashlib import unittest -from apache_beam.internal import code_object_pickler from parameterized import parameterized +from apache_beam.internal import code_object_pickler + def top_level_function(): return 1 From 3f25da37fb649b298c2f02ade5831abb2288642e Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Wed, 6 Aug 2025 13:08:19 -0400 Subject: [PATCH 122/142] Update code_object_pickler.py --- .../internal/code_object_pickler.py | 57 ++++++++++++++----- 1 file changed, 44 insertions(+), 13 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler.py b/sdks/python/apache_beam/internal/code_object_pickler.py index 7daa572bbdb4..1f0f74ed0440 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler.py +++ b/sdks/python/apache_beam/internal/code_object_pickler.py @@ -58,14 +58,23 @@ def get_normalized_path(path): return path -def _get_code_path(callable): +def get_code_path(callable: types.FunctionType): """Returns the stable reference to the code object. + Will be implemented using cloudpickle in a future version. + Args: callable: The callable object to search for. Returns: The stable reference to the code object. + Examples: + - __main__.top_level_function.__code__ + - __main__.ClassWithNestedFunction.process.__code__.co_consts[nested_function] + - __main__.ClassWithNestedLambda.process.__code__.co_consts[ + get_lambda_from_dictionary].co_consts[, ('x',)] + - __main__.ClassWithNestedLambda.process.__code__.co_consts[ + , ('x',), 1234567890] """ code_path = _extend_path( callable.__module__, @@ -78,7 +87,7 @@ def _get_code_path(callable): return code_path -def _extend_path(prefix, suffix): +def _extend_path(prefix: str, suffix: str): """Extends the path to the code object. Args: @@ -95,13 +104,17 @@ def _extend_path(prefix, suffix): return prefix + '.' + suffix -def _search(callable, node, qual_name_parts): +def _search( + callable: types.FunctionType, node: object, qual_name_parts: list[str] +): """Searches an object to create a stable reference code path. Recursively searches the tree of objects starting from node and looking for callable and returns a string to uniquely identify the path from node to the callable. + Example of qual_name_parts: ['ClassWithLambda', 'process', '', ''] + Args: callable: The callable object to search for. node: The object to search within. @@ -125,7 +138,9 @@ def _search(callable, node, qual_name_parts): return _search_code(callable, node, qual_name_parts) -def _search_module_or_class(callable, node, qual_name_parts): +def _search_module_or_class( + callable: types.FunctionType, node: object, qual_name_parts: list[str] +): """Searches a module or class to create a stable reference code path. Args: @@ -161,7 +176,11 @@ def _search_module_or_class(callable, node, qual_name_parts): first_part, _search(callable, getattr(node, first_part), rest)) -def _search_function(callable, node, qual_name_parts): +def _search_function( + callable: types.FunctionType, + node: types.FunctionType, + qual_name_parts: list[str] +): """Searches a function to create a stable reference code path. Args: @@ -185,7 +204,11 @@ def _search_function(callable, node, qual_name_parts): '__code__', _search(callable, node.__code__, qual_name_parts)) -def _search_code(callable, node, qual_name_parts): +def _search_code( + callable: types.FunctionType, + node: types.CodeType, + qual_name_parts: list[str] +): """Searches a code object to create a stable reference code path. Args: @@ -224,7 +247,11 @@ def _search_code(callable, node, qual_name_parts): return _search(callable, node, rest) -def _search_lambda(callable, code_objects_by_name, qual_name_parts): +def _search_lambda( + callable: types.FunctionType, + code_objects_by_name: dict[str, list[types.CodeType]], + qual_name_parts: list[str] +): """Searches a lambda to create a stable reference code path. Args: @@ -284,7 +311,9 @@ def _search_lambda(callable, code_objects_by_name, qual_name_parts): _ARGUMENT_PATTERN = re.compile(r"'([^']*)'") -def _get_code_object_from_single_name_pattern(obj, name_result, path): +def _get_code_object_from_single_name_pattern( + obj: types.CodeType, name_result: re.Match[str], path: str +): """Returns the code object from a name pattern. Args: @@ -309,7 +338,8 @@ def _get_code_object_from_single_name_pattern(obj, name_result, path): def _get_code_object_from_lambda_with_args_pattern( - obj, lambda_with_args_result, path): + obj: types.CodeType, lambda_with_args_result: re.Match[str], path: str +): """Returns the code object from a lambda with args pattern. Args: @@ -335,7 +365,8 @@ def _get_code_object_from_lambda_with_args_pattern( def _get_code_object_from_lambda_with_hash_pattern( - obj, lambda_with_hash_result, path): + obj: types.CodeType, lambda_with_hash_result: re.Match[str], path: str +): """Returns the code object from a lambda with hash pattern. Args: @@ -362,7 +393,7 @@ def _get_code_object_from_lambda_with_hash_pattern( raise AttributeError(f'Could not find code object with path: {path}') -def _get_code_from_stable_reference(path): +def _get_code_from_stable_reference(path: str): """Returns the code object from a stable reference. Args: @@ -400,7 +431,7 @@ def _get_code_from_stable_reference(path): return obj -def _signature(obj): +def _signature(obj: types.CodeType): """Returns the signature of a code object. The signature is the names of the arguments of the code object. This is used @@ -423,7 +454,7 @@ def _signature(obj): return None -def _create_bytecode_hash(code_object): +def _create_bytecode_hash(code_object: types.CodeType): """Returns the hash of a code object. Args: From bd63ea5cda2079e4a593bc556c122eef5bc8cdaa Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Wed, 6 Aug 2025 13:43:12 -0400 Subject: [PATCH 123/142] Update code_object_pickler_test.py --- sdks/python/apache_beam/internal/code_object_pickler_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index 6332ad97f4fc..b61aadbcde22 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -192,7 +192,7 @@ def get_lambda_from_dictionary(): class CodeObjectPicklerTest(unittest.TestCase): @parameterized.expand(test_cases) def test_get_code_path(self, callable, expected): - actual = code_object_pickler._get_code_path(callable) + actual = code_object_pickler.get_code_path(callable) self.assertEqual(actual, expected) @parameterized.expand(test_cases) From 922ea3a3b99bcae45f20179eeb4b400d8ff3e5c1 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Wed, 6 Aug 2025 13:44:22 -0400 Subject: [PATCH 124/142] Update pickle_code_path_test.py --- .../internal/pickle_code_path_test.py | 102 +++++++++--------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/sdks/python/apache_beam/internal/pickle_code_path_test.py b/sdks/python/apache_beam/internal/pickle_code_path_test.py index 8953ddf2e374..10de7e5decf6 100644 --- a/sdks/python/apache_beam/internal/pickle_code_path_test.py +++ b/sdks/python/apache_beam/internal/pickle_code_path_test.py @@ -72,7 +72,7 @@ def test_get_code_from_stable_reference_invalid_lambda_with_hash_path_raises_exc def test_get_code_add_local_variable_in_class_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_classes.AddLocalVariable.my_method( self)).replace( "before_module_with_classes", @@ -83,7 +83,7 @@ def test_get_code_add_local_variable_in_class_objects_match(self): def test_get_code_remove_local_variable_in_class_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_classes.RemoveLocalVariable.my_method( self)).replace( "before_module_with_classes", @@ -94,7 +94,7 @@ def test_get_code_remove_local_variable_in_class_objects_match(self): def test_get_code_add_lambda_variable_in_class_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_classes.AddLambdaVariable.my_method( self)).replace( "before_module_with_classes", @@ -105,7 +105,7 @@ def test_get_code_add_lambda_variable_in_class_objects_match(self): def test_get_code_remove_lambda_variable_in_class_objects_match(self): with self.assertRaisesRegex(AttributeError, "object has no attribute"): code_object_pickler._get_code_from_stable_reference( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_classes.RemoveLambdaVariable.my_method( self)).replace( "before_module_with_classes", @@ -114,7 +114,7 @@ def test_get_code_remove_lambda_variable_in_class_objects_match(self): def test_get_code_nested_function_in_class_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_classes.ClassWithNestedFunction.my_method( self)).replace( "before_module_with_classes", @@ -126,7 +126,7 @@ def test_get_code_nested_function_in_class_objects_match(self): def test_get_code_nested_function_2_in_class_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_classes.ClassWithNestedFunction2.my_method( self)).replace( "before_module_with_classes", @@ -138,7 +138,7 @@ def test_get_code_nested_function_2_in_class_objects_match(self): def test_get_code_add_new_function_in_class_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_classes.ClassWithTwoMethods.my_method( self)).replace( "before_module_with_classes", @@ -149,7 +149,7 @@ def test_get_code_add_new_function_in_class_objects_match(self): def test_get_code_remove_method_in_class_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_classes.RemoveMethod.my_method( self)).replace( "before_module_with_classes", @@ -160,7 +160,7 @@ def test_get_code_remove_method_in_class_objects_match(self): def test_get_code_add_global_variable_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_functions.my_function()).replace( "before_module_with_functions", "after_module_with_global_variable", @@ -171,7 +171,7 @@ def test_get_code_add_global_variable_objects_match(self): def test_get_code_add_top_level_function_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_functions.my_function()).replace( "before_module_with_functions", "after_module_add_function")), @@ -181,7 +181,7 @@ def test_get_code_add_top_level_function_objects_match(self): def test_get_code_add_local_variable_in_function_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_functions.my_function()).replace( "before_module_with_functions", "after_module_add_variable")), @@ -191,7 +191,7 @@ def test_get_code_add_local_variable_in_function_objects_match(self): def test_get_code_remove_local_variable_in_function_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_functions.my_function()).replace( "before_module_with_functions", "after_module_remove_variable")), @@ -201,7 +201,7 @@ def test_get_code_remove_local_variable_in_function_objects_match(self): def test_get_code_nested_function_in_function_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_functions.my_function()).replace( "before_module_with_functions", "after_module_with_nested_function")), @@ -211,7 +211,7 @@ def test_get_code_nested_function_in_function_objects_match(self): def test_get_code_nested_function_2_in_function_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_functions.my_function()).replace( "before_module_with_functions", "after_module_with_nested_function_2")), @@ -221,7 +221,7 @@ def test_get_code_nested_function_2_in_function_objects_match(self): def test_get_code_add_class_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_functions.my_function()).replace( "before_module_with_functions", "after_module_with_single_class")), @@ -231,7 +231,7 @@ def test_get_code_add_class_objects_match(self): def test_get_code_add_lambda_variable_in_function_objects_match(self): self.assertEqual( code_object_pickler._get_code_from_stable_reference( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_functions.my_function()).replace( "before_module_with_functions", "after_module_add_lambda_variable")), @@ -241,177 +241,177 @@ def test_get_code_add_lambda_variable_in_function_objects_match(self): def test_get_code_remove_lambda_variable_in_function_raises_exception(self): with self.assertRaisesRegex(AttributeError, "object has no attribute"): code_object_pickler._get_code_from_stable_reference( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_lambdas.my_function()).replace( "before_module_with_lambdas", "after_module_remove_lambda_variable")) def test_identifiers_new_local_variable_in_class_paths_match(self): self.assertEqual( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_classes.AddLocalVariable.my_method( self)).replace("before_module_with_classes", "module_name"), - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( after_module_with_classes.AddLocalVariable.my_method(self)).replace( "after_module_with_classes", "module_name"), ) def test_identifiers_remove_local_variable_in_class_paths_match(self): self.assertEqual( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_classes.RemoveLocalVariable.my_method( self)).replace("before_module_with_classes", "module_name"), - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( after_module_with_classes.RemoveLocalVariable.my_method( self)).replace("after_module_with_classes", "module_name"), ) def test_identifiers_add_lambda_variable_in_class_paths_match(self): self.assertNotEqual( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_classes.AddLambdaVariable.my_method( self)).replace("before_module_with_classes", "module_name"), - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( after_module_with_classes.AddLambdaVariable.my_method( self)).replace("after_module_with_classes", "module_name"), ) def test_identifiers_remove_lambda_variable_in_class_paths_match(self): self.assertNotEqual( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_classes.RemoveLambdaVariable.my_method( self)).replace("before_module_with_classes", "module_name"), - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( after_module_with_classes.RemoveLambdaVariable.my_method( self)).replace("after_module_with_classes", "module_name"), ) def test_identifiers_nested_function_in_class_paths_match(self): self.assertEqual( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_classes.ClassWithNestedFunction.my_method( self)).replace("before_module_with_classes", "module_name"), - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( after_module_with_classes.ClassWithNestedFunction.my_method( self)).replace("after_module_with_classes", "module_name"), ) def test_identifiers_nested_function_2_in_class_paths_match(self): self.assertEqual( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_classes.ClassWithNestedFunction2.my_method( self)).replace("before_module_with_classes", "module_name"), - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( after_module_with_classes.ClassWithNestedFunction2.my_method( self)).replace("after_module_with_classes", "module_name"), ) def test_identifiers_add_new_function_in_class_paths_match(self): self.assertEqual( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_classes.ClassWithTwoMethods.my_method( self)).replace("before_module_with_classes", "module_name"), - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( after_module_with_classes.ClassWithTwoMethods.my_method( self)).replace("after_module_with_classes", "module_name"), ) def test_identifiers_remove_function_in_class_paths_match(self): self.assertEqual( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_classes.RemoveMethod.my_method(self)).replace( "before_module_with_classes", "module_name"), - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( after_module_with_classes.RemoveMethod.my_method(self)).replace( "after_module_with_classes", "module_name"), ) def test_identifiers_add_global_variable_paths_match(self): self.assertEqual( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_functions.my_function()).replace( "before_module_with_functions", "module_name"), - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( after_module_with_global_variable.my_function()).replace( "after_module_with_global_variable", "module_name"), ) def test_identifiers_add_top_level_function_paths_match(self): self.assertEqual( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_functions.my_function()).replace( "before_module_with_functions", "module_name"), - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( after_module_add_function.my_function()).replace( "after_module_add_function", "module_name"), ) def test_identifiers_add_local_variable_in_function_paths_match(self): self.assertEqual( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_functions.my_function()).replace( "before_module_with_functions", "module_name"), - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( after_module_add_variable.my_function()).replace( "after_module_add_variable", "module_name"), ) def test_identifiers_remove_local_variable_in_function_paths_match(self): self.assertEqual( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_functions.my_function()).replace( "before_module_with_functions", "module_name"), - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( after_module_remove_variable.my_function()).replace( "after_module_remove_variable", "module_name"), ) def test_identifiers_nested_function_in_function_paths_match(self): self.assertEqual( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_functions.my_function()).replace( "before_module_with_functions", "module_name"), - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( after_module_with_nested_function.my_function()).replace( "after_module_with_nested_function", "module_name"), ) def test_identifiers_nested_function_2_in_function_paths_match(self): self.assertEqual( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_functions.my_function()).replace( "before_module_with_functions", "module_name"), - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( after_module_with_nested_function_2.my_function()).replace( "after_module_with_nested_function_2", "module_name"), ) def test_identifiers_add_class_paths_match(self): self.assertEqual( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_functions.my_function()).replace( "before_module_with_functions", "module_name"), - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( after_module_with_single_class.my_function()).replace( "after_module_with_single_class", "module_name"), ) def test_identifiers_add_lambda_variable_in_function_no_match(self): self.assertNotEqual( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_functions.my_function()).replace( "before_module_with_functions", "module_name"), - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( after_module_add_lambda_variable.my_function()).replace( "after_module_add_lambda_variable", "module_name"), ) def test_identifiers_remove_lambda_variable_in_function_no_match(self): self.assertNotEqual( - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( before_module_with_lambdas.my_function()).replace( "before_module_with_lambdas", "module_name"), - code_object_pickler._get_code_path( + code_object_pickler.get_code_path( after_module_remove_lambda_variable.my_function()).replace( "after_module_remove_lambda_variable", "module_name"), ) From b82e78f60845ae12238b8a335ecb245ef738094a Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Wed, 6 Aug 2025 14:19:31 -0400 Subject: [PATCH 125/142] fix formatting and lint --- .../internal/code_object_pickler.py | 29 +++++++------------ .../internal/code_object_pickler_test.py | 2 +- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler.py b/sdks/python/apache_beam/internal/code_object_pickler.py index 1f0f74ed0440..18dc378db0f6 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler.py +++ b/sdks/python/apache_beam/internal/code_object_pickler.py @@ -70,7 +70,8 @@ def get_code_path(callable: types.FunctionType): The stable reference to the code object. Examples: - __main__.top_level_function.__code__ - - __main__.ClassWithNestedFunction.process.__code__.co_consts[nested_function] + - __main__.ClassWithNestedFunction.process.__code__.co_consts[ + nested_function] - __main__.ClassWithNestedLambda.process.__code__.co_consts[ get_lambda_from_dictionary].co_consts[, ('x',)] - __main__.ClassWithNestedLambda.process.__code__.co_consts[ @@ -105,15 +106,14 @@ def _extend_path(prefix: str, suffix: str): def _search( - callable: types.FunctionType, node: object, qual_name_parts: list[str] -): + callable: types.FunctionType, node: object, qual_name_parts: list[str]): """Searches an object to create a stable reference code path. Recursively searches the tree of objects starting from node and looking for callable and returns a string to uniquely identify the path from node to the callable. - Example of qual_name_parts: ['ClassWithLambda', 'process', '', ''] + Example of qual_name_parts: ['MyClass', 'process', '', ''] Args: callable: The callable object to search for. @@ -139,8 +139,7 @@ def _search( def _search_module_or_class( - callable: types.FunctionType, node: object, qual_name_parts: list[str] -): + callable: types.FunctionType, node: object, qual_name_parts: list[str]): """Searches a module or class to create a stable reference code path. Args: @@ -179,8 +178,7 @@ def _search_module_or_class( def _search_function( callable: types.FunctionType, node: types.FunctionType, - qual_name_parts: list[str] -): + qual_name_parts: list[str]): """Searches a function to create a stable reference code path. Args: @@ -207,8 +205,7 @@ def _search_function( def _search_code( callable: types.FunctionType, node: types.CodeType, - qual_name_parts: list[str] -): + qual_name_parts: list[str]): """Searches a code object to create a stable reference code path. Args: @@ -250,8 +247,7 @@ def _search_code( def _search_lambda( callable: types.FunctionType, code_objects_by_name: dict[str, list[types.CodeType]], - qual_name_parts: list[str] -): + qual_name_parts: list[str]): """Searches a lambda to create a stable reference code path. Args: @@ -312,8 +308,7 @@ def _search_lambda( def _get_code_object_from_single_name_pattern( - obj: types.CodeType, name_result: re.Match[str], path: str -): + obj: types.CodeType, name_result: re.Match[str], path: str): """Returns the code object from a name pattern. Args: @@ -338,8 +333,7 @@ def _get_code_object_from_single_name_pattern( def _get_code_object_from_lambda_with_args_pattern( - obj: types.CodeType, lambda_with_args_result: re.Match[str], path: str -): + obj: types.CodeType, lambda_with_args_result: re.Match[str], path: str): """Returns the code object from a lambda with args pattern. Args: @@ -365,8 +359,7 @@ def _get_code_object_from_lambda_with_args_pattern( def _get_code_object_from_lambda_with_hash_pattern( - obj: types.CodeType, lambda_with_hash_result: re.Match[str], path: str -): + obj: types.CodeType, lambda_with_hash_result: re.Match[str], path: str): """Returns the code object from a lambda with hash pattern. Args: diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index b61aadbcde22..0420c6137ab7 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -202,7 +202,7 @@ def test_get_code_from_stable_reference(self, callable, path): @parameterized.expand(test_cases) def test_roundtrip(self, callable, _): - path = code_object_pickler._get_code_path(callable) + path = code_object_pickler.get_code_path(callable) actual = code_object_pickler._get_code_from_stable_reference(path) self.assertEqual(actual, callable.__code__) From 1e4428028c3338a39bc041fd020fe22bb328a126 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Wed, 6 Aug 2025 14:54:26 -0400 Subject: [PATCH 126/142] formatting and lint changes --- .../internal/code_object_pickler.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler.py b/sdks/python/apache_beam/internal/code_object_pickler.py index 18dc378db0f6..ae5a0c12caf9 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler.py +++ b/sdks/python/apache_beam/internal/code_object_pickler.py @@ -106,7 +106,7 @@ def _extend_path(prefix: str, suffix: str): def _search( - callable: types.FunctionType, node: object, qual_name_parts: list[str]): + callable: types.FunctionType, node: object, qual_name_parts: list[str]): """Searches an object to create a stable reference code path. Recursively searches the tree of objects starting from node and looking for @@ -139,7 +139,7 @@ def _search( def _search_module_or_class( - callable: types.FunctionType, node: object, qual_name_parts: list[str]): + callable: types.FunctionType, node: object, qual_name_parts: list[str]): """Searches a module or class to create a stable reference code path. Args: @@ -176,9 +176,9 @@ def _search_module_or_class( def _search_function( - callable: types.FunctionType, - node: types.FunctionType, - qual_name_parts: list[str]): + callable: types.FunctionType, + node: types.FunctionType, + qual_name_parts: list[str]): """Searches a function to create a stable reference code path. Args: @@ -203,9 +203,9 @@ def _search_function( def _search_code( - callable: types.FunctionType, - node: types.CodeType, - qual_name_parts: list[str]): + callable: types.FunctionType, + node: types.CodeType, + qual_name_parts: list[str]): """Searches a code object to create a stable reference code path. Args: @@ -245,9 +245,9 @@ def _search_code( def _search_lambda( - callable: types.FunctionType, - code_objects_by_name: dict[str, list[types.CodeType]], - qual_name_parts: list[str]): + callable: types.FunctionType, + code_objects_by_name: dict[str, list[types.CodeType]], + qual_name_parts: list[str]): """Searches a lambda to create a stable reference code path. Args: @@ -308,7 +308,7 @@ def _search_lambda( def _get_code_object_from_single_name_pattern( - obj: types.CodeType, name_result: re.Match[str], path: str): + obj: Module, name_result: re.Match[str], path: str): """Returns the code object from a name pattern. Args: @@ -333,7 +333,7 @@ def _get_code_object_from_single_name_pattern( def _get_code_object_from_lambda_with_args_pattern( - obj: types.CodeType, lambda_with_args_result: re.Match[str], path: str): + obj: Module, lambda_with_args_result: re.Match[str], path: str): """Returns the code object from a lambda with args pattern. Args: @@ -359,7 +359,7 @@ def _get_code_object_from_lambda_with_args_pattern( def _get_code_object_from_lambda_with_hash_pattern( - obj: types.CodeType, lambda_with_hash_result: re.Match[str], path: str): + obj: Module, lambda_with_hash_result: re.Match[str], path: str): """Returns the code object from a lambda with hash pattern. Args: From 211c0f4dfb0370bdb74ae110c96a2a4341053c3c Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Wed, 6 Aug 2025 14:59:28 -0400 Subject: [PATCH 127/142] Update code_object_pickler.py --- sdks/python/apache_beam/internal/code_object_pickler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler.py b/sdks/python/apache_beam/internal/code_object_pickler.py index ae5a0c12caf9..259517cd6d61 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler.py +++ b/sdks/python/apache_beam/internal/code_object_pickler.py @@ -308,7 +308,7 @@ def _search_lambda( def _get_code_object_from_single_name_pattern( - obj: Module, name_result: re.Match[str], path: str): + obj: types.ModuleType, name_result: re.Match[str], path: str): """Returns the code object from a name pattern. Args: @@ -333,7 +333,7 @@ def _get_code_object_from_single_name_pattern( def _get_code_object_from_lambda_with_args_pattern( - obj: Module, lambda_with_args_result: re.Match[str], path: str): + obj: types.ModuleType, lambda_with_args_result: re.Match[str], path: str): """Returns the code object from a lambda with args pattern. Args: @@ -359,7 +359,7 @@ def _get_code_object_from_lambda_with_args_pattern( def _get_code_object_from_lambda_with_hash_pattern( - obj: Module, lambda_with_hash_result: re.Match[str], path: str): + obj: types.ModuleType, lambda_with_hash_result: re.Match[str], path: str): """Returns the code object from a lambda with hash pattern. Args: From cd83d963a7285e264f220070cbcc8ee32f1d4955 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Thu, 7 Aug 2025 11:12:47 -0400 Subject: [PATCH 128/142] change type hints --- .../internal/code_object_pickler.py | 76 +++++++++++++------ 1 file changed, 52 insertions(+), 24 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler.py b/sdks/python/apache_beam/internal/code_object_pickler.py index 259517cd6d61..73a9e2042245 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler.py +++ b/sdks/python/apache_beam/internal/code_object_pickler.py @@ -51,6 +51,7 @@ import re import sys import types +from typing import Any, Callable def get_normalized_path(path): @@ -58,7 +59,7 @@ def get_normalized_path(path): return path -def get_code_path(callable: types.FunctionType): +def get_code_path(callable: Callable): """Returns the stable reference to the code object. Will be implemented using cloudpickle in a future version. @@ -77,6 +78,9 @@ def get_code_path(callable: types.FunctionType): - __main__.ClassWithNestedLambda.process.__code__.co_consts[ , ('x',), 1234567890] """ + if not hasattr(callable, '__module__') or not hasattr( + callable, '__qualname__'): + return None code_path = _extend_path( callable.__module__, _search( @@ -106,19 +110,22 @@ def _extend_path(prefix: str, suffix: str): def _search( - callable: types.FunctionType, node: object, qual_name_parts: list[str]): + callable: Callable, node: Any, qual_name_parts: list[str]): """Searches an object to create a stable reference code path. - Recursively searches the tree of objects starting from node and looking for - callable and returns a string to uniquely identify the path from node to the - callable. + Recursively searches the tree of objects starting from node to find the + callable's code object. It uses qual_name_parts to navigate through + attributes. Special components like '' and '' direct the + search within nested code objects. + Example of qual_name_parts: ['MyClass', 'process', '', ''] Args: callable: The callable object to search for. node: The object to search within. - qual_name_parts: A list of strings representing the qualified name. + qual_name_parts: A list of strings representing the qualified name of the + callable object. Returns: The stable reference to the code object, or None if not found. @@ -126,7 +133,9 @@ def _search( if node is None: return None if not qual_name_parts: - if hasattr(node, '__code__') and node.__code__ == callable.__code__: + if (hasattr(node, '__code__') + and hasattr(callable, '__code__') + and node.__code__ == callable.__code__): return '__code__' else: return None @@ -139,7 +148,7 @@ def _search( def _search_module_or_class( - callable: types.FunctionType, node: object, qual_name_parts: list[str]): + callable: Callable, node: Any, qual_name_parts: list[str]): """Searches a module or class to create a stable reference code path. Args: @@ -159,8 +168,9 @@ def _search_module_or_class( if first_part == '': for name in dir(node): value = getattr(node, name) - if (isinstance(value, type(callable)) and - value.__code__ == callable.__code__): + if (hasattr(callable, '__code__') + and isinstance(value, type(callable)) + and value.__code__ == callable.__code__): return name + '.__code__' elif (isinstance(value, types.FunctionType) and value.__defaults__ is not None): @@ -175,10 +185,7 @@ def _search_module_or_class( first_part, _search(callable, getattr(node, first_part), rest)) -def _search_function( - callable: types.FunctionType, - node: types.FunctionType, - qual_name_parts: list[str]): +def _search_function(callable: Callable, node: Any, qual_name_parts: list[str]): """Searches a function to create a stable reference code path. Args: @@ -190,6 +197,9 @@ def _search_function( The stable reference to the code object, or None if not found. """ first_part = qual_name_parts[0] + if (hasattr(callable, '__code__') + and hasattr(node, '__code__') + and node.__code__ == callable.__code__): if node.__code__ == callable.__code__: if len(qual_name_parts) > 1: raise ValueError('Qual name parts too long') @@ -202,10 +212,7 @@ def _search_function( '__code__', _search(callable, node.__code__, qual_name_parts)) -def _search_code( - callable: types.FunctionType, - node: types.CodeType, - qual_name_parts: list[str]): +def _search_code(callable: Callable, node: Any, qual_name_parts: list[str]): """Searches a code object to create a stable reference code path. Args: @@ -221,7 +228,7 @@ def _search_code( """ first_part = qual_name_parts[0] rest = qual_name_parts[1:] - if node == callable.__code__: + if hasattr(callable, '__code__') and node == callable.__code__: if len(qual_name_parts) > 1: raise ValueError('Qual name parts too long') return '' @@ -245,7 +252,7 @@ def _search_code( def _search_lambda( - callable: types.FunctionType, + callable: Callable, code_objects_by_name: dict[str, list[types.CodeType]], qual_name_parts: list[str]): """Searches a lambda to create a stable reference code path. @@ -308,7 +315,7 @@ def _search_lambda( def _get_code_object_from_single_name_pattern( - obj: types.ModuleType, name_result: re.Match[str], path: str): + obj: Callable, name_result: re.Match[str], path: str): """Returns the code object from a name pattern. Args: @@ -321,7 +328,11 @@ def _get_code_object_from_single_name_pattern( Raises: ValueError: If the pattern is invalid. + AttributeError: If the code object is not found or the object does not have + the co_consts attribute. """ + if not hasattr(obj, 'co_consts'): + raise AttributeError(f'Object {obj} has no co_consts attribute') if len(name_result.groups()) > 1: raise ValueError(f'Invalid pattern for single name: {name_result.group(0)}') # Groups are indexed starting at 1, group(0) is the entire match. @@ -333,7 +344,7 @@ def _get_code_object_from_single_name_pattern( def _get_code_object_from_lambda_with_args_pattern( - obj: types.ModuleType, lambda_with_args_result: re.Match[str], path: str): + obj: Callable, lambda_with_args_result: re.Match[str], path: str): """Returns the code object from a lambda with args pattern. Args: @@ -343,7 +354,13 @@ def _get_code_object_from_lambda_with_args_pattern( Returns: The code object. + + Raises: + AttributeError: If the code object is not found or the object does not have + the co_consts attribute. """ + if not hasattr(obj, 'co_consts'): + raise AttributeError(f'Object {obj} has no co_consts attribute') name = lambda_with_args_result.group(1) code_objects = collections.defaultdict(list) for co_const in obj.co_consts: @@ -359,7 +376,7 @@ def _get_code_object_from_lambda_with_args_pattern( def _get_code_object_from_lambda_with_hash_pattern( - obj: types.ModuleType, lambda_with_hash_result: re.Match[str], path: str): + obj: Callable, lambda_with_hash_result: re.Match[str], path: str): """Returns the code object from a lambda with hash pattern. Args: @@ -369,7 +386,13 @@ def _get_code_object_from_lambda_with_hash_pattern( Returns: The code object. + + Raises: + AttributeError: If the code object is not found or the object does not have + the co_consts attribute. """ + if not hasattr(obj, 'co_consts'): + raise AttributeError(f'Object {obj} has no co_consts attribute') name = lambda_with_hash_result.group(1) code_objects = collections.defaultdict(list) for co_const in obj.co_consts: @@ -424,7 +447,7 @@ def _get_code_from_stable_reference(path: str): return obj -def _signature(obj: types.CodeType): +def _signature(obj: Callable): """Returns the signature of a code object. The signature is the names of the arguments of the code object. This is used @@ -455,5 +478,10 @@ def _create_bytecode_hash(code_object: types.CodeType): Returns: The hash of the code object. + + Raises: + TypeError: If the object given is not a code object. """ + if not isinstance(code_object, types.CodeType): + raise TypeError(f'{code_object} is not a code object') return hashlib.md5(code_object.co_code).hexdigest() From 9660d413b41433cf9f9ea3d9ff4b297528f7851a Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Thu, 7 Aug 2025 11:28:45 -0400 Subject: [PATCH 129/142] fix typo that caused error --- sdks/python/apache_beam/internal/code_object_pickler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler.py b/sdks/python/apache_beam/internal/code_object_pickler.py index 73a9e2042245..5f645ac7e3b2 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler.py +++ b/sdks/python/apache_beam/internal/code_object_pickler.py @@ -200,7 +200,6 @@ def _search_function(callable: Callable, node: Any, qual_name_parts: list[str]): if (hasattr(callable, '__code__') and hasattr(node, '__code__') and node.__code__ == callable.__code__): - if node.__code__ == callable.__code__: if len(qual_name_parts) > 1: raise ValueError('Qual name parts too long') return '__code__' From 6410707ee853b32a85a41647a35a6dd36d68501c Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Thu, 7 Aug 2025 13:53:13 -0400 Subject: [PATCH 130/142] fix formatting and lint errors --- .../internal/code_object_pickler.py | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler.py b/sdks/python/apache_beam/internal/code_object_pickler.py index 5f645ac7e3b2..ad7bbf1c422b 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler.py +++ b/sdks/python/apache_beam/internal/code_object_pickler.py @@ -78,8 +78,8 @@ def get_code_path(callable: Callable): - __main__.ClassWithNestedLambda.process.__code__.co_consts[ , ('x',), 1234567890] """ - if not hasattr(callable, '__module__') or not hasattr( - callable, '__qualname__'): + if not hasattr(callable, '__module__') or not hasattr(callable, + '__qualname__'): return None code_path = _extend_path( callable.__module__, @@ -109,8 +109,7 @@ def _extend_path(prefix: str, suffix: str): return prefix + '.' + suffix -def _search( - callable: Callable, node: Any, qual_name_parts: list[str]): +def _search(callable: Callable, node: Any, qual_name_parts: list[str]): """Searches an object to create a stable reference code path. Recursively searches the tree of objects starting from node to find the @@ -133,9 +132,8 @@ def _search( if node is None: return None if not qual_name_parts: - if (hasattr(node, '__code__') - and hasattr(callable, '__code__') - and node.__code__ == callable.__code__): + if (hasattr(node, '__code__') and hasattr(callable, '__code__') and + node.__code__ == callable.__code__): return '__code__' else: return None @@ -168,9 +166,9 @@ def _search_module_or_class( if first_part == '': for name in dir(node): value = getattr(node, name) - if (hasattr(callable, '__code__') - and isinstance(value, type(callable)) - and value.__code__ == callable.__code__): + if (hasattr(callable, '__code__') and + isinstance(value, type(callable)) and + value.__code__ == callable.__code__): return name + '.__code__' elif (isinstance(value, types.FunctionType) and value.__defaults__ is not None): @@ -197,9 +195,8 @@ def _search_function(callable: Callable, node: Any, qual_name_parts: list[str]): The stable reference to the code object, or None if not found. """ first_part = qual_name_parts[0] - if (hasattr(callable, '__code__') - and hasattr(node, '__code__') - and node.__code__ == callable.__code__): + if (hasattr(callable, '__code__') and hasattr(node, '__code__') and + node.__code__ == callable.__code__): if len(qual_name_parts) > 1: raise ValueError('Qual name parts too long') return '__code__' @@ -252,7 +249,7 @@ def _search_code(callable: Callable, node: Any, qual_name_parts: list[str]): def _search_lambda( callable: Callable, - code_objects_by_name: dict[str, list[types.CodeType]], + code_objects_by_name: dict[str, list[Callable]], qual_name_parts: list[str]): """Searches a lambda to create a stable reference code path. @@ -424,7 +421,7 @@ def _get_code_from_stable_reference(path: str): if not path: raise ValueError('Path must not be empty.') parts = path.split('.') - obj = sys.modules[parts[0]] + obj: Callable = sys.modules[parts[0]] for part in parts[1:]: if name_result := _SINGLE_NAME_PATTERN.fullmatch(part): obj = _get_code_object_from_single_name_pattern(obj, name_result, path) From d49f850642fe02dba5f9c30895e07e3ab6d22e1c Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Thu, 7 Aug 2025 16:54:19 -0400 Subject: [PATCH 131/142] fix typehints --- .../internal/code_object_pickler.py | 60 ++++++++----------- 1 file changed, 24 insertions(+), 36 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler.py b/sdks/python/apache_beam/internal/code_object_pickler.py index ad7bbf1c422b..6110f8e36b02 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler.py +++ b/sdks/python/apache_beam/internal/code_object_pickler.py @@ -183,7 +183,10 @@ def _search_module_or_class( first_part, _search(callable, getattr(node, first_part), rest)) -def _search_function(callable: Callable, node: Any, qual_name_parts: list[str]): +def _search_function( + callable: types.FunctionType, + node: types.FunctionType, + qual_name_parts: list[str]): """Searches a function to create a stable reference code path. Args: @@ -195,8 +198,7 @@ def _search_function(callable: Callable, node: Any, qual_name_parts: list[str]): The stable reference to the code object, or None if not found. """ first_part = qual_name_parts[0] - if (hasattr(callable, '__code__') and hasattr(node, '__code__') and - node.__code__ == callable.__code__): + if (node.__code__ == callable.__code__): if len(qual_name_parts) > 1: raise ValueError('Qual name parts too long') return '__code__' @@ -208,7 +210,10 @@ def _search_function(callable: Callable, node: Any, qual_name_parts: list[str]): '__code__', _search(callable, node.__code__, qual_name_parts)) -def _search_code(callable: Callable, node: Any, qual_name_parts: list[str]): +def _search_code( + callable: types.FunctionType, + node: types.CodeType, + qual_name_parts: list[str]): """Searches a code object to create a stable reference code path. Args: @@ -249,7 +254,7 @@ def _search_code(callable: Callable, node: Any, qual_name_parts: list[str]): def _search_lambda( callable: Callable, - code_objects_by_name: dict[str, list[Callable]], + code_objects_by_name: dict[str, list[types.CodeType]], qual_name_parts: list[str]): """Searches a lambda to create a stable reference code path. @@ -311,7 +316,7 @@ def _search_lambda( def _get_code_object_from_single_name_pattern( - obj: Callable, name_result: re.Match[str], path: str): + obj: types.CodeType, name_result: re.Match[str], path: str): """Returns the code object from a name pattern. Args: @@ -324,11 +329,8 @@ def _get_code_object_from_single_name_pattern( Raises: ValueError: If the pattern is invalid. - AttributeError: If the code object is not found or the object does not have - the co_consts attribute. + AttributeError: If the code object is not found. """ - if not hasattr(obj, 'co_consts'): - raise AttributeError(f'Object {obj} has no co_consts attribute') if len(name_result.groups()) > 1: raise ValueError(f'Invalid pattern for single name: {name_result.group(0)}') # Groups are indexed starting at 1, group(0) is the entire match. @@ -340,7 +342,7 @@ def _get_code_object_from_single_name_pattern( def _get_code_object_from_lambda_with_args_pattern( - obj: Callable, lambda_with_args_result: re.Match[str], path: str): + obj: types.CodeType, lambda_with_args_result: re.Match[str], path: str): """Returns the code object from a lambda with args pattern. Args: @@ -352,11 +354,8 @@ def _get_code_object_from_lambda_with_args_pattern( The code object. Raises: - AttributeError: If the code object is not found or the object does not have - the co_consts attribute. + AttributeError: If the code object is not found. """ - if not hasattr(obj, 'co_consts'): - raise AttributeError(f'Object {obj} has no co_consts attribute') name = lambda_with_args_result.group(1) code_objects = collections.defaultdict(list) for co_const in obj.co_consts: @@ -372,7 +371,7 @@ def _get_code_object_from_lambda_with_args_pattern( def _get_code_object_from_lambda_with_hash_pattern( - obj: Callable, lambda_with_hash_result: re.Match[str], path: str): + obj: types.CodeType, lambda_with_hash_result: re.Match[str], path: str): """Returns the code object from a lambda with hash pattern. Args: @@ -384,11 +383,8 @@ def _get_code_object_from_lambda_with_hash_pattern( The code object. Raises: - AttributeError: If the code object is not found or the object does not have - the co_consts attribute. + AttributeError: If the code object is not found. """ - if not hasattr(obj, 'co_consts'): - raise AttributeError(f'Object {obj} has no co_consts attribute') name = lambda_with_hash_result.group(1) code_objects = collections.defaultdict(list) for co_const in obj.co_consts: @@ -421,7 +417,7 @@ def _get_code_from_stable_reference(path: str): if not path: raise ValueError('Path must not be empty.') parts = path.split('.') - obj: Callable = sys.modules[parts[0]] + obj = sys.modules[parts[0]] for part in parts[1:]: if name_result := _SINGLE_NAME_PATTERN.fullmatch(part): obj = _get_code_object_from_single_name_pattern(obj, name_result, path) @@ -443,7 +439,7 @@ def _get_code_from_stable_reference(path: str): return obj -def _signature(obj: Callable): +def _signature(obj: types.CodeType): """Returns the signature of a code object. The signature is the names of the arguments of the code object. This is used @@ -455,15 +451,12 @@ def _signature(obj: Callable): Returns: A tuple of the names of the arguments of the code object. """ - if isinstance(obj, types.CodeType): - arg_count = ( - obj.co_argcount + obj.co_kwonlyargcount + - (obj.co_flags & 4 == 4) # PyCF_VARARGS - + (obj.co_flags & 8 == 8) # PyCF_VARKEYWORDS - ) - return obj.co_varnames[:arg_count] - else: - return None + arg_count = ( + obj.co_argcount + obj.co_kwonlyargcount + + (obj.co_flags & 4 == 4) # PyCF_VARARGS + + (obj.co_flags & 8 == 8) # PyCF_VARKEYWORDS + ) + return obj.co_varnames[:arg_count] def _create_bytecode_hash(code_object: types.CodeType): @@ -474,10 +467,5 @@ def _create_bytecode_hash(code_object: types.CodeType): Returns: The hash of the code object. - - Raises: - TypeError: If the object given is not a code object. """ - if not isinstance(code_object, types.CodeType): - raise TypeError(f'{code_object} is not a code object') return hashlib.md5(code_object.co_code).hexdigest() From 8f4344b055da578953b9fee1b125785294cae08b Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Fri, 8 Aug 2025 16:43:25 -0400 Subject: [PATCH 132/142] fix formatting and lint --- .../internal/code_object_pickler.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler.py b/sdks/python/apache_beam/internal/code_object_pickler.py index 6110f8e36b02..6feb33509dcb 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler.py +++ b/sdks/python/apache_beam/internal/code_object_pickler.py @@ -109,7 +109,7 @@ def _extend_path(prefix: str, suffix: str): return prefix + '.' + suffix -def _search(callable: Callable, node: Any, qual_name_parts: list[str]): +def _search(callable: Any, node: Any, qual_name_parts: list[str]): """Searches an object to create a stable reference code path. Recursively searches the tree of objects starting from node to find the @@ -184,9 +184,9 @@ def _search_module_or_class( def _search_function( - callable: types.FunctionType, - node: types.FunctionType, - qual_name_parts: list[str]): + callable: types.FunctionType, + node: types.FunctionType, + qual_name_parts: list[str]): """Searches a function to create a stable reference code path. Args: @@ -211,9 +211,9 @@ def _search_function( def _search_code( - callable: types.FunctionType, - node: types.CodeType, - qual_name_parts: list[str]): + callable: types.FunctionType, + node: types.CodeType, + qual_name_parts: list[str]): """Searches a code object to create a stable reference code path. Args: @@ -316,7 +316,7 @@ def _search_lambda( def _get_code_object_from_single_name_pattern( - obj: types.CodeType, name_result: re.Match[str], path: str): + obj: types.ModuleType, name_result: re.Match[str], path: str): """Returns the code object from a name pattern. Args: @@ -342,7 +342,7 @@ def _get_code_object_from_single_name_pattern( def _get_code_object_from_lambda_with_args_pattern( - obj: types.CodeType, lambda_with_args_result: re.Match[str], path: str): + obj: types.ModuleType, lambda_with_args_result: re.Match[str], path: str): """Returns the code object from a lambda with args pattern. Args: @@ -371,7 +371,7 @@ def _get_code_object_from_lambda_with_args_pattern( def _get_code_object_from_lambda_with_hash_pattern( - obj: types.CodeType, lambda_with_hash_result: re.Match[str], path: str): + obj: types.ModuleType, lambda_with_hash_result: re.Match[str], path: str): """Returns the code object from a lambda with hash pattern. Args: From a4ebbd5696f96aeafdecb7f3e931262b33e0e77f Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Mon, 11 Aug 2025 10:27:09 -0400 Subject: [PATCH 133/142] Update code_object_pickler.py --- sdks/python/apache_beam/internal/code_object_pickler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler.py b/sdks/python/apache_beam/internal/code_object_pickler.py index 6feb33509dcb..3e56ae92ea61 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler.py +++ b/sdks/python/apache_beam/internal/code_object_pickler.py @@ -51,7 +51,8 @@ import re import sys import types -from typing import Any, Callable +from typing import Any +from typing import Callable def get_normalized_path(path): From bac70ebc3ffebd9528cae5d906389d76da7588ef Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 12 Aug 2025 15:31:01 -0400 Subject: [PATCH 134/142] Update code_object_pickler.py --- .../apache_beam/internal/code_object_pickler.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler.py b/sdks/python/apache_beam/internal/code_object_pickler.py index 3e56ae92ea61..ce1d364dd00c 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler.py +++ b/sdks/python/apache_beam/internal/code_object_pickler.py @@ -51,8 +51,6 @@ import re import sys import types -from typing import Any -from typing import Callable def get_normalized_path(path): @@ -60,7 +58,7 @@ def get_normalized_path(path): return path -def get_code_path(callable: Callable): +def get_code_path(callable: types.FunctionType): """Returns the stable reference to the code object. Will be implemented using cloudpickle in a future version. @@ -110,7 +108,10 @@ def _extend_path(prefix: str, suffix: str): return prefix + '.' + suffix -def _search(callable: Any, node: Any, qual_name_parts: list[str]): +def _search( + callable: types.FunctionType, + node: types.ModuleType | types.FunctionType | types.CodeType, + qual_name_parts: list[str]): """Searches an object to create a stable reference code path. Recursively searches the tree of objects starting from node to find the @@ -147,7 +148,9 @@ def _search(callable: Any, node: Any, qual_name_parts: list[str]): def _search_module_or_class( - callable: Callable, node: Any, qual_name_parts: list[str]): + callable: types.FunctionType, + node: types.ModuleType, + qual_name_parts: list[str]): """Searches a module or class to create a stable reference code path. Args: @@ -254,7 +257,7 @@ def _search_code( def _search_lambda( - callable: Callable, + callable: types.FunctionType, code_objects_by_name: dict[str, list[types.CodeType]], qual_name_parts: list[str]): """Searches a lambda to create a stable reference code path. From 7be70e66d04210de02b594a426704b12229ae305 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 12 Aug 2025 16:03:57 -0400 Subject: [PATCH 135/142] fix union error --- sdks/python/apache_beam/internal/code_object_pickler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler.py b/sdks/python/apache_beam/internal/code_object_pickler.py index ce1d364dd00c..5ec89af22c7d 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler.py +++ b/sdks/python/apache_beam/internal/code_object_pickler.py @@ -110,7 +110,7 @@ def _extend_path(prefix: str, suffix: str): def _search( callable: types.FunctionType, - node: types.ModuleType | types.FunctionType | types.CodeType, + node: Union[types.ModuleType, types.FunctionType, types.CodeType], qual_name_parts: list[str]): """Searches an object to create a stable reference code path. From eca023dbe90951ae1096c2e6eba2d1fcfb260ca9 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Tue, 12 Aug 2025 16:08:37 -0400 Subject: [PATCH 136/142] import union --- sdks/python/apache_beam/internal/code_object_pickler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdks/python/apache_beam/internal/code_object_pickler.py b/sdks/python/apache_beam/internal/code_object_pickler.py index 5ec89af22c7d..b62305882474 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler.py +++ b/sdks/python/apache_beam/internal/code_object_pickler.py @@ -51,6 +51,7 @@ import re import sys import types +from typing import Union def get_normalized_path(path): From ff59041c08e5ca5ac532c67f140116769f108e2d Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Thu, 14 Aug 2025 11:47:04 -0400 Subject: [PATCH 137/142] update code_object_pickler.py docstring, move pickle_code_path_test.py into code_object_pickler_test.py, rename files in test_data, rename tests in code_object_pickler_test.py --- .../internal/code_object_pickler.py | 57 +-- .../internal/code_object_pickler_test.py | 411 ++++++++++++++++- .../internal/pickle_code_path_test.py | 421 ------------------ .../{test_cases => test_data}/__init__.py | 2 +- .../module_1.py} | 0 .../module_1_class_added.py} | 0 .../module_1_function_added.py} | 0 .../module_1_global_variable_added.py} | 0 .../module_1_lambda_variable_added.py} | 0 .../module_1_local_variable_added.py} | 0 .../module_1_local_variable_removed.py} | 0 .../module_1_nested_function_2_added.py} | 0 .../module_1_nested_function_added.py} | 0 .../module_2.py} | 0 .../module_2_modified.py} | 0 .../module_3.py} | 0 .../module_3_modified.py} | 0 .../module_with_default_argument.py | 0 18 files changed, 437 insertions(+), 454 deletions(-) delete mode 100644 sdks/python/apache_beam/internal/pickle_code_path_test.py rename sdks/python/apache_beam/internal/{test_cases => test_data}/__init__.py (89%) rename sdks/python/apache_beam/internal/{test_cases/before_module_with_functions.py => test_data/module_1.py} (100%) rename sdks/python/apache_beam/internal/{test_cases/after_module_with_single_class.py => test_data/module_1_class_added.py} (100%) rename sdks/python/apache_beam/internal/{test_cases/after_module_add_function.py => test_data/module_1_function_added.py} (100%) rename sdks/python/apache_beam/internal/{test_cases/after_module_with_global_variable.py => test_data/module_1_global_variable_added.py} (100%) rename sdks/python/apache_beam/internal/{test_cases/after_module_add_lambda_variable.py => test_data/module_1_lambda_variable_added.py} (100%) rename sdks/python/apache_beam/internal/{test_cases/after_module_add_variable.py => test_data/module_1_local_variable_added.py} (100%) rename sdks/python/apache_beam/internal/{test_cases/after_module_remove_variable.py => test_data/module_1_local_variable_removed.py} (100%) rename sdks/python/apache_beam/internal/{test_cases/after_module_with_nested_function.py => test_data/module_1_nested_function_2_added.py} (100%) rename sdks/python/apache_beam/internal/{test_cases/after_module_with_nested_function_2.py => test_data/module_1_nested_function_added.py} (100%) rename sdks/python/apache_beam/internal/{test_cases/before_module_with_classes.py => test_data/module_2.py} (100%) rename sdks/python/apache_beam/internal/{test_cases/after_module_with_classes.py => test_data/module_2_modified.py} (100%) rename sdks/python/apache_beam/internal/{test_cases/before_module_with_lambdas.py => test_data/module_3.py} (100%) rename sdks/python/apache_beam/internal/{test_cases/after_module_remove_lambda_variable.py => test_data/module_3_modified.py} (100%) rename sdks/python/apache_beam/internal/{test_cases => test_data}/module_with_default_argument.py (100%) diff --git a/sdks/python/apache_beam/internal/code_object_pickler.py b/sdks/python/apache_beam/internal/code_object_pickler.py index b62305882474..397bf8f1079c 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler.py +++ b/sdks/python/apache_beam/internal/code_object_pickler.py @@ -17,10 +17,14 @@ """Customizations to how Python code objects are pickled. -This module provides functions for pickling code objects, especially lambdas, -in a consistent way. It addresses issues with non-deterministic pickling by -creating a unique identifier that is invariant to small changes in the source -code. +This module provides helper functions to improve pickling code objects, +especially lambdas, in a consistent way by using code object identifiers. These +helper functions will be used to patch pickler implementations used by Beam +(e.g. Cloudpickle). + +A code object identifier is a unique identifier for a code object that provides +a unique reference to the code object in the context where the code is defined +and is invariant to small changes in the currounding code. The code object identifiers consists of a sequence of the following parts separated by periods: @@ -51,6 +55,7 @@ import re import sys import types +from typing import Optional from typing import Union @@ -60,15 +65,13 @@ def get_normalized_path(path): def get_code_path(callable: types.FunctionType): - """Returns the stable reference to the code object. - - Will be implemented using cloudpickle in a future version. + """Returns the code object identifier for a given callable. Args: callable: The callable object to search for. Returns: - The stable reference to the code object. + The code object identifier. Examples: - __main__.top_level_function.__code__ - __main__.ClassWithNestedFunction.process.__code__.co_consts[ @@ -81,7 +84,7 @@ def get_code_path(callable: types.FunctionType): if not hasattr(callable, '__module__') or not hasattr(callable, '__qualname__'): return None - code_path = _extend_path( + code_path: str = _extend_path( callable.__module__, _search( callable, @@ -92,7 +95,7 @@ def get_code_path(callable: types.FunctionType): return code_path -def _extend_path(prefix: str, suffix: str): +def _extend_path(prefix: str, current_path: Optional[str]): """Extends the path to the code object. Args: @@ -113,12 +116,14 @@ def _search( callable: types.FunctionType, node: Union[types.ModuleType, types.FunctionType, types.CodeType], qual_name_parts: list[str]): - """Searches an object to create a stable reference code path. + """Searches an object to create a code object identifier. Recursively searches the tree of objects starting from node to find the - callable's code object. It uses qual_name_parts to navigate through - attributes. Special components like '' and '' direct the - search within nested code objects. + callable's code object. It navigates through the attributes by using + the first element of qual_name_parts to indicate what object it is + currently at, then recursively passes through the rest of the list until + the callable is found. Special components like '' and '' + direct the search within nested code objects. Example of qual_name_parts: ['MyClass', 'process', '', ''] @@ -130,7 +135,7 @@ def _search( callable object. Returns: - The stable reference to the code object, or None if not found. + The code object identifier, or None if not found. """ if node is None: return None @@ -152,7 +157,7 @@ def _search_module_or_class( callable: types.FunctionType, node: types.ModuleType, qual_name_parts: list[str]): - """Searches a module or class to create a stable reference code path. + """Searches a module or class to create a code object identifier. Args: callable: The callable object to search for. @@ -160,7 +165,7 @@ def _search_module_or_class( qual_name_parts: The list of qual name parts. Returns: - The stable reference to the code object, or None if not found. + The code object identifier, or None if not found. """ # Functions/methods have a name that is unique within a given module or class # so the traversal can directly lookup function object identified by the name. @@ -192,7 +197,7 @@ def _search_function( callable: types.FunctionType, node: types.FunctionType, qual_name_parts: list[str]): - """Searches a function to create a stable reference code path. + """Searches a function to create a code object identifier. Args: callable: The callable object to search for. @@ -200,7 +205,7 @@ def _search_function( qual_name_parts: The list of qual name parts. Returns: - The stable reference to the code object, or None if not found. + The code object identifier, or None if not found. """ first_part = qual_name_parts[0] if (node.__code__ == callable.__code__): @@ -219,7 +224,7 @@ def _search_code( callable: types.FunctionType, node: types.CodeType, qual_name_parts: list[str]): - """Searches a code object to create a stable reference code path. + """Searches a code object to create a code object identifier. Args: callable: The callable to search for. @@ -227,7 +232,7 @@ def _search_code( qual_name_parts: The list of qual name parts. Returns: - The stable reference to the code object, or None if not found. + The code object identifier, or None if not found. Raises: ValueError: If the qual name parts are too long. @@ -261,7 +266,7 @@ def _search_lambda( callable: types.FunctionType, code_objects_by_name: dict[str, list[types.CodeType]], qual_name_parts: list[str]): - """Searches a lambda to create a stable reference code path. + """Searches a lambda to create a code object identifier. Args: callable: The callable to search for. @@ -269,7 +274,7 @@ def _search_lambda( qual_name_parts: The rest of the qual_name_parts. Returns: - The stable reference to the code object, or None if not found. + The code object identifier, or None if not found. """ # There are multiple lambdas in the code object, so we need to calculate # the signature and the hash to identify the correct lambda. @@ -406,11 +411,11 @@ def _get_code_object_from_lambda_with_hash_pattern( raise AttributeError(f'Could not find code object with path: {path}') -def _get_code_from_stable_reference(path: str): - """Returns the code object from a stable reference. +def get_code_from_stable_reference(path: str): + """Returns the code object from a stable reference code object identifier. Args: - path: A string representing the stable reference to the code object. + path: A string representing the code object identifier. Returns: The code object. diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index 0420c6137ab7..5239dac87040 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -15,12 +15,30 @@ # limitations under the License. # +"""Tests for generating stable identifiers to use for Pickle serialization.""" + import hashlib import unittest from parameterized import parameterized +# pylint: disable=unused-import +from apache_beam.internal import code_object_pickler from apache_beam.internal import code_object_pickler +from apache_beam.internal.test_data import module_1_class_added +from apache_beam.internal.test_data import module_1_function_added +from apache_beam.internal.test_data import module_1_global_variable_added +from apache_beam.internal.test_data import module_1_lambda_variable_added +from apache_beam.internal.test_data import module_1_local_variable_added +from apache_beam.internal.test_data import module_1_local_variable_removed +from apache_beam.internal.test_data import module_1_nested_function_2_added +from apache_beam.internal.test_data import module_1_nested_function_added +from apache_beam.internal.test_data import module_1 +from apache_beam.internal.test_data import module_2_modified +from apache_beam.internal.test_data import module_2 +from apache_beam.internal.test_data import module_3_modified +from apache_beam.internal.test_data import module_3 +from apache_beam.internal.test_data import module_with_default_argument def top_level_function(): @@ -189,23 +207,404 @@ def get_lambda_from_dictionary(): ] -class CodeObjectPicklerTest(unittest.TestCase): +class CodePathGenerationTest(unittest.TestCase): @parameterized.expand(test_cases) - def test_get_code_path(self, callable, expected): + def test_get_code_path(self, callable, expected_path): actual = code_object_pickler.get_code_path(callable) self.assertEqual(actual, expected) @parameterized.expand(test_cases) - def test_get_code_from_stable_reference(self, callable, path): - actual = code_object_pickler._get_code_from_stable_reference(path) + def test_get_code_from_stable_reference(self, expected_callable, path): + actual = code_object_pickler.get_code_from_stable_reference(path) self.assertEqual(actual, callable.__code__) @parameterized.expand(test_cases) - def test_roundtrip(self, callable, _): + def test_roundtrip(self, callable, unused_path): path = code_object_pickler.get_code_path(callable) - actual = code_object_pickler._get_code_from_stable_reference(path) + actual = code_object_pickler.get_code_from_stable_reference(path) self.assertEqual(actual, callable.__code__) +class GetCodeFromStableReferenceTest(unittest.TestCase): + def empty_path_raises_exception(self): + with self.assertRaisesRegex(ValueError, "Path must not be empty"): + code_object_pickler.get_code_from_stable_reference("") + + def invalid_default_index_raises_exception(self): + with self.assertRaisesRegex(ValueError, "out of bounds"): + code_object_pickler.get_code_from_stable_reference( + "apache_beam.internal.test_cases.module_with_default_argument." + "function_with_lambda_default_argument.__defaults__[1]") + + def invalid_single_name_path_raises_exception(self): + with self.assertRaisesRegex(AttributeError, + "Could not find code object with path"): + code_object_pickler.get_code_from_stable_reference( + "apache_beam.internal.test_cases.module_3." + "my_function.__code__.co_consts[something]") + + def invalid_lambda_with_args_path_raises_exception(self): + with self.assertRaisesRegex(AttributeError, + "Could not find code object with path"): + code_object_pickler.get_code_from_stable_reference( + "apache_beam.internal.test_cases.module_3." + "my_function.__code__.co_consts[, ('x',)]") + + def invalid_lambda_with_hash_path_raises_exception(self): + with self.assertRaisesRegex(AttributeError, + "Could not find code object with path"): + code_object_pickler.get_code_from_stable_reference( + "apache_beam.internal.test_cases.module_3." + "my_function.__code__.co_consts[, ('',), 1234567890]") + + def test_adding_local_variable_in_class_preserves_object(self): + self.assertEqual( + code_object_pickler.get_code_from_stable_reference( + code_object_pickler.get_code_path( + module_2.AddLocalVariable.my_method( + self)).replace( + "module_2", + "module_2_modified")), + module_2_modified.AddLocalVariable.my_method(self).__code__, + ) + + def test_removing_local_variable_in_class_preserves_object(self): + self.assertEqual( + code_object_pickler.get_code_from_stable_reference( + code_object_pickler.get_code_path( + module_2.RemoveLocalVariable.my_method( + self)).replace( + "module_2", + "module_2_modified")), + module_2_modified.RemoveLocalVariable.my_method(self).__code__, + ) + + def test_adding_lambda_variable_in_class_preserves_object(self): + self.assertEqual( + code_object_pickler.get_code_from_stable_reference( + code_object_pickler.get_code_path( + module_2.AddLambdaVariable.my_method( + self)).replace( + "module_2", + "module_2_modified")), + module_2_modified.AddLambdaVariable.my_method(self).__code__, + ) + + def test_removing_lambda_variable_in_class_changes_object(self): + with self.assertRaisesRegex(AttributeError, "object has no attribute"): + code_object_pickler.( + code_object_pickler.get_code_path( + module_2.RemoveLambdaVariable.my_method( + self)).replace( + "module_2", + "module_2_modified")) + + def test_adding_nested_function_in_class_preserves_object(self): + self.assertEqual( + code_object_pickler.get_code_from_stable_reference( + code_object_pickler.get_code_path( + module_2.ClassWithNestedFunction.my_method( + self)).replace( + "module_2", + "module_2_modified")), + module_2_modified.ClassWithNestedFunction.my_method( + self).__code__, + ) + + def test_adding_nested_function_2_in_class_preserves_object(self): + self.assertEqual( + code_object_pickler.get_code_from_stable_reference( + code_object_pickler.get_code_path( + module_2.ClassWithNestedFunction2.my_method( + self)).replace( + "module_2", + "module_2_modified")), + module_2_modified.ClassWithNestedFunction2.my_method( + self).__code__, + ) + + def test_adding_new_function_in_class_preserves_object(self): + self.assertEqual( + code_object_pickler.get_code_from_stable_reference( + code_object_pickler.get_code_path( + module_2.ClassWithTwoMethods.my_method( + self)).replace( + "module_2", + "module_2_modified")), + module_2_modified.ClassWithTwoMethods.my_method(self).__code__, + ) + + def test_removing_method_in_class_preserves_object(self): + self.assertEqual( + code_object_pickler.get_code_from_stable_reference( + code_object_pickler.get_code_path( + module_2.RemoveMethod.my_method( + self)).replace( + "module_2", + "module_2_modified")), + module_2_modified.RemoveMethod.my_method(self).__code__, + ) + + def test_adding_global_variable_preserves_object(self): + self.assertEqual( + code_object_pickler.get_code_from_stable_reference( + code_object_pickler.get_code_path( + module_1.my_function()).replace( + "module_1", + "module_1_global_variable_added", + )), + module_1_global_variable_added.my_function().__code__, + ) + + def test_adding_top_level_function_preserves_object(self): + self.assertEqual( + code_object_pickler.get_code_from_stable_reference( + code_object_pickler.get_code_path( + module_1.my_function()).replace( + "module_1", + "module_1_function_added")), + module_1_function_added.my_function().__code__, + ) + + def test_adding_local_variable_in_function_preserves_object(self): + self.assertEqual( + code_object_pickler.get_code_from_stable_reference( + code_object_pickler.get_code_path( + module_1.my_function()).replace( + "module_1", + "module_1_local_variable_added")), + module_1_local_variable_added.my_function().__code__, + ) + + def test_removing_local_variable_in_function_preserves_object(self): + self.assertEqual( + code_object_pickler.get_code_from_stable_reference( + code_object_pickler.get_code_path( + module_1.my_function()).replace( + "module_1", + "module_1_local_variable_removed")), + module_1_local_variable_removed.my_function().__code__, + ) + + def test_adding_nested_function_in_function_preserves_object(self): + self.assertEqual( + code_object_pickler.get_code_from_stable_reference( + code_object_pickler.get_code_path( + module_1.my_function()).replace( + "module_1", + "module_1_nested_function_added")), + module_1_nested_function_added.my_function().__code__, + ) + + def test_adding_nested_function_2_in_function_preserves_object(self): + self.assertEqual( + code_object_pickler.get_code_from_stable_reference( + code_object_pickler.get_code_path( + module_1.my_function()).replace( + "module_1", + "module_1_nested_function_2_added")), + module_1_nested_function_2_added.my_function().__code__, + ) + + def test_adding_class_preserves_object(self): + self.assertEqual( + code_object_pickler.get_code_from_stable_reference( + code_object_pickler.get_code_path( + module_1.my_function()).replace( + "module_1", + "module_1_class_added")), + module_1_class_added.my_function().__code__, + ) + + def test_adding_lambda_variable_in_function_preserves_object(self): + self.assertEqual( + code_object_pickler.get_code_from_stable_reference( + code_object_pickler.get_code_path( + module_1.my_function()).replace( + "module_1", + "module_1_lambda_variable_added")), + module_1_lambda_variable_added.my_function().__code__, + ) + + def test_removing_lambda_variable_in_function_raises_exception(self): + with self.assertRaisesRegex(AttributeError, "object has no attribute"): + code_object_pickler.get_code_from_stable_reference( + code_object_pickler.get_code_path( + module_3.my_function()).replace( + "module_3", + "module_3_modified")) + + +class CodePathStabilityTest(unittest.TestCase): + def test_adding_local_variable_in_class_preserves_path(self): + self.assertEqual( + code_object_pickler.get_code_path( + module_2.AddLocalVariable.my_method( + self)).replace("module_2", "module_name"), + code_object_pickler.get_code_path( + module_2_modified.AddLocalVariable.my_method(self)).replace( + "module_2_modified", "module_name"), + ) + + def test_removing_local_variable_in_class_preserves_path(self): + self.assertEqual( + code_object_pickler.get_code_path( + module_2.RemoveLocalVariable.my_method( + self)).replace("module_2", "module_name"), + code_object_pickler.get_code_path( + module_2_modified.RemoveLocalVariable.my_method( + self)).replace("module_2_modified", "module_name"), + ) + + def test_adding_lambda_variable_in_class_changes_path(self): + self.assertNotEqual( + code_object_pickler.get_code_path( + module_2.AddLambdaVariable.my_method( + self)).replace("module_2", "module_name"), + code_object_pickler.get_code_path( + module_2_modified.AddLambdaVariable.my_method( + self)).replace("module_2_modified", "module_name"), + ) + + def test_removing_lambda_variable_in_class_changes_path(self): + self.assertNotEqual( + code_object_pickler.get_code_path( + module_2.RemoveLambdaVariable.my_method( + self)).replace("module_2", "module_name"), + code_object_pickler.get_code_path( + module_2_modified.RemoveLambdaVariable.my_method( + self)).replace("module_2_modified", "module_name"), + ) + + def test_adding_nested_function_in_class_preserves_path(self): + self.assertEqual( + code_object_pickler.get_code_path( + module_2.ClassWithNestedFunction.my_method( + self)).replace("module_2", "module_name"), + code_object_pickler.get_code_path( + module_2_modified.ClassWithNestedFunction.my_method( + self)).replace("module_2_modified", "module_name"), + ) + + def test_adding_nested_function_2_in_class_preserves_path(self): + self.assertEqual( + code_object_pickler.get_code_path( + module_2.ClassWithNestedFunction2.my_method( + self)).replace("module_2", "module_name"), + code_object_pickler.get_code_path( + module_2_modified.ClassWithNestedFunction2.my_method( + self)).replace("module_2_modified", "module_name"), + ) + + def test_adding_new_function_in_class_preserves_path(self): + self.assertEqual( + code_object_pickler.get_code_path( + module_2.ClassWithTwoMethods.my_method( + self)).replace("module_2", "module_name"), + code_object_pickler.get_code_path( + module_2_modified.ClassWithTwoMethods.my_method( + self)).replace("module_2_modified", "module_name"), + ) + + def test_removing_function_in_class_preserves_path(self): + self.assertEqual( + code_object_pickler.get_code_path( + module_2.RemoveMethod.my_method(self)).replace( + "module_2", "module_name"), + code_object_pickler.get_code_path( + module_2_modified.RemoveMethod.my_method(self)).replace( + "module_2_modified", "module_name"), + ) + + def test_adding_global_variable_preserves_path(self): + self.assertEqual( + code_object_pickler.get_code_path( + module_1.my_function()).replace( + "module_1", "module_name"), + code_object_pickler.get_code_path( + module_1_global_variable_added.my_function()).replace( + "module_1_global_variable_added", "module_name"), + ) + + def test_adding_top_level_function_preserves_path(self): + self.assertEqual( + code_object_pickler.get_code_path( + module_1.my_function()).replace( + "module_1", "module_name"), + code_object_pickler.get_code_path( + module_1_function_added.my_function()).replace( + "module_1_function_added", "module_name"), + ) + + def test_adding_local_variable_in_function_preserves_path(self): + self.assertEqual( + code_object_pickler.get_code_path( + module_1.my_function()).replace( + "module_1", "module_name"), + code_object_pickler.get_code_path( + module_1_local_variable_added.my_function()).replace( + "module_1_local_variable_added", "module_name"), + ) + + def test_removing_local_variable_in_function_preserves_path(self): + self.assertEqual( + code_object_pickler.get_code_path( + module_1.my_function()).replace( + "module_1", "module_name"), + code_object_pickler.get_code_path( + module_1_local_variable_removed.my_function()).replace( + "module_1_local_variable_removed", "module_name"), + ) + + def test_adding_nested_function_in_function_preserves_path(self): + self.assertEqual( + code_object_pickler.get_code_path( + module_1.my_function()).replace( + "module_1", "module_name"), + code_object_pickler.get_code_path( + module_1_nested_function_added.my_function()).replace( + "module_1_nested_function_added", "module_name"), + ) + + def test_adding_nested_function_2_in_function_preserves_path(self): + self.assertEqual( + code_object_pickler.get_code_path( + module_1.my_function()).replace( + "module_1", "module_name"), + code_object_pickler.get_code_path( + module_1_nested_function_2_added.my_function()).replace( + "module_1_nested_function_2_added", "module_name"), + ) + + def test_adding_class_preserves_path(self): + self.assertEqual( + code_object_pickler.get_code_path( + module_1.my_function()).replace( + "module_1", "module_name"), + code_object_pickler.get_code_path( + module_1_class_added.my_function()).replace( + "module_1_class_added", "module_name"), + ) + + def test_adding_lambda_variable_in_function_changes_path(self): + self.assertNotEqual( + code_object_pickler.get_code_path( + module_1.my_function()).replace( + "module_1", "module_name"), + code_object_pickler.get_code_path( + module_1_lambda_variable_added.my_function()).replace( + "module_1_lambda_variable_added", "module_name"), + ) + + def test_removing_lambda_variable_in_function_changes_path(self): + self.assertNotEqual( + code_object_pickler.get_code_path( + module_3.my_function()).replace( + "module_3", "module_name"), + code_object_pickler.get_code_path( + module_3_modified.my_function()).replace( + "module_3_modified", "module_name"), + ) + if __name__ == "__main__": unittest.main() diff --git a/sdks/python/apache_beam/internal/pickle_code_path_test.py b/sdks/python/apache_beam/internal/pickle_code_path_test.py deleted file mode 100644 index 10de7e5decf6..000000000000 --- a/sdks/python/apache_beam/internal/pickle_code_path_test.py +++ /dev/null @@ -1,421 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -"""Tests for generating stable identifiers to use for Pickle serialization.""" - -import unittest - -# pylint: disable=unused-import -from apache_beam.internal import code_object_pickler -from apache_beam.internal.test_cases import after_module_add_function -from apache_beam.internal.test_cases import after_module_add_lambda_variable -from apache_beam.internal.test_cases import after_module_add_variable -from apache_beam.internal.test_cases import after_module_remove_lambda_variable -from apache_beam.internal.test_cases import after_module_remove_variable -from apache_beam.internal.test_cases import after_module_with_classes -from apache_beam.internal.test_cases import after_module_with_global_variable -from apache_beam.internal.test_cases import after_module_with_nested_function -from apache_beam.internal.test_cases import after_module_with_nested_function_2 -from apache_beam.internal.test_cases import after_module_with_single_class -from apache_beam.internal.test_cases import before_module_with_classes -from apache_beam.internal.test_cases import before_module_with_functions -from apache_beam.internal.test_cases import before_module_with_lambdas -from apache_beam.internal.test_cases import module_with_default_argument - - -class CodePathTest(unittest.TestCase): - def test_get_code_from_stable_reference_empty_path_raises_exception(self): - with self.assertRaisesRegex(ValueError, "Path must not be empty"): - code_object_pickler._get_code_from_stable_reference("") - - def test_get_code_from_stable_reference_invalid_default_index_raises_exception(self): # pylint: disable=line-too-long - with self.assertRaisesRegex(ValueError, "out of bounds"): - code_object_pickler._get_code_from_stable_reference( - "apache_beam.internal.test_cases.module_with_default_argument." - "function_with_lambda_default_argument.__defaults__[1]") - - def test_get_code_from_stable_reference_invalid_single_name_path_raises_exception(self): # pylint: disable=line-too-long - with self.assertRaisesRegex(AttributeError, - "Could not find code object with path"): - code_object_pickler._get_code_from_stable_reference( - "apache_beam.internal.test_cases.before_module_with_lambdas." - "my_function.__code__.co_consts[something]") - - def test_get_code_from_stable_reference_invalid_lambda_with_args_path_raises_exception(self): # pylint: disable=line-too-long - with self.assertRaisesRegex(AttributeError, - "Could not find code object with path"): - code_object_pickler._get_code_from_stable_reference( - "apache_beam.internal.test_cases.before_module_with_lambdas." - "my_function.__code__.co_consts[, ('x',)]") - - def test_get_code_from_stable_reference_invalid_lambda_with_hash_path_raises_exception(self): # pylint: disable=line-too-long - with self.assertRaisesRegex(AttributeError, - "Could not find code object with path"): - code_object_pickler._get_code_from_stable_reference( - "apache_beam.internal.test_cases.before_module_with_lambdas." - "my_function.__code__.co_consts[, ('',), 1234567890]") - - def test_get_code_add_local_variable_in_class_objects_match(self): - self.assertEqual( - code_object_pickler._get_code_from_stable_reference( - code_object_pickler.get_code_path( - before_module_with_classes.AddLocalVariable.my_method( - self)).replace( - "before_module_with_classes", - "after_module_with_classes")), - after_module_with_classes.AddLocalVariable.my_method(self).__code__, - ) - - def test_get_code_remove_local_variable_in_class_objects_match(self): - self.assertEqual( - code_object_pickler._get_code_from_stable_reference( - code_object_pickler.get_code_path( - before_module_with_classes.RemoveLocalVariable.my_method( - self)).replace( - "before_module_with_classes", - "after_module_with_classes")), - after_module_with_classes.RemoveLocalVariable.my_method(self).__code__, - ) - - def test_get_code_add_lambda_variable_in_class_objects_match(self): - self.assertEqual( - code_object_pickler._get_code_from_stable_reference( - code_object_pickler.get_code_path( - before_module_with_classes.AddLambdaVariable.my_method( - self)).replace( - "before_module_with_classes", - "after_module_with_classes")), - after_module_with_classes.AddLambdaVariable.my_method(self).__code__, - ) - - def test_get_code_remove_lambda_variable_in_class_objects_match(self): - with self.assertRaisesRegex(AttributeError, "object has no attribute"): - code_object_pickler._get_code_from_stable_reference( - code_object_pickler.get_code_path( - before_module_with_classes.RemoveLambdaVariable.my_method( - self)).replace( - "before_module_with_classes", - "after_module_with_classes")) - - def test_get_code_nested_function_in_class_objects_match(self): - self.assertEqual( - code_object_pickler._get_code_from_stable_reference( - code_object_pickler.get_code_path( - before_module_with_classes.ClassWithNestedFunction.my_method( - self)).replace( - "before_module_with_classes", - "after_module_with_classes")), - after_module_with_classes.ClassWithNestedFunction.my_method( - self).__code__, - ) - - def test_get_code_nested_function_2_in_class_objects_match(self): - self.assertEqual( - code_object_pickler._get_code_from_stable_reference( - code_object_pickler.get_code_path( - before_module_with_classes.ClassWithNestedFunction2.my_method( - self)).replace( - "before_module_with_classes", - "after_module_with_classes")), - after_module_with_classes.ClassWithNestedFunction2.my_method( - self).__code__, - ) - - def test_get_code_add_new_function_in_class_objects_match(self): - self.assertEqual( - code_object_pickler._get_code_from_stable_reference( - code_object_pickler.get_code_path( - before_module_with_classes.ClassWithTwoMethods.my_method( - self)).replace( - "before_module_with_classes", - "after_module_with_classes")), - after_module_with_classes.ClassWithTwoMethods.my_method(self).__code__, - ) - - def test_get_code_remove_method_in_class_objects_match(self): - self.assertEqual( - code_object_pickler._get_code_from_stable_reference( - code_object_pickler.get_code_path( - before_module_with_classes.RemoveMethod.my_method( - self)).replace( - "before_module_with_classes", - "after_module_with_classes")), - after_module_with_classes.RemoveMethod.my_method(self).__code__, - ) - - def test_get_code_add_global_variable_objects_match(self): - self.assertEqual( - code_object_pickler._get_code_from_stable_reference( - code_object_pickler.get_code_path( - before_module_with_functions.my_function()).replace( - "before_module_with_functions", - "after_module_with_global_variable", - )), - after_module_with_global_variable.my_function().__code__, - ) - - def test_get_code_add_top_level_function_objects_match(self): - self.assertEqual( - code_object_pickler._get_code_from_stable_reference( - code_object_pickler.get_code_path( - before_module_with_functions.my_function()).replace( - "before_module_with_functions", - "after_module_add_function")), - after_module_add_function.my_function().__code__, - ) - - def test_get_code_add_local_variable_in_function_objects_match(self): - self.assertEqual( - code_object_pickler._get_code_from_stable_reference( - code_object_pickler.get_code_path( - before_module_with_functions.my_function()).replace( - "before_module_with_functions", - "after_module_add_variable")), - after_module_add_variable.my_function().__code__, - ) - - def test_get_code_remove_local_variable_in_function_objects_match(self): - self.assertEqual( - code_object_pickler._get_code_from_stable_reference( - code_object_pickler.get_code_path( - before_module_with_functions.my_function()).replace( - "before_module_with_functions", - "after_module_remove_variable")), - after_module_remove_variable.my_function().__code__, - ) - - def test_get_code_nested_function_in_function_objects_match(self): - self.assertEqual( - code_object_pickler._get_code_from_stable_reference( - code_object_pickler.get_code_path( - before_module_with_functions.my_function()).replace( - "before_module_with_functions", - "after_module_with_nested_function")), - after_module_with_nested_function.my_function().__code__, - ) - - def test_get_code_nested_function_2_in_function_objects_match(self): - self.assertEqual( - code_object_pickler._get_code_from_stable_reference( - code_object_pickler.get_code_path( - before_module_with_functions.my_function()).replace( - "before_module_with_functions", - "after_module_with_nested_function_2")), - after_module_with_nested_function_2.my_function().__code__, - ) - - def test_get_code_add_class_objects_match(self): - self.assertEqual( - code_object_pickler._get_code_from_stable_reference( - code_object_pickler.get_code_path( - before_module_with_functions.my_function()).replace( - "before_module_with_functions", - "after_module_with_single_class")), - after_module_with_single_class.my_function().__code__, - ) - - def test_get_code_add_lambda_variable_in_function_objects_match(self): - self.assertEqual( - code_object_pickler._get_code_from_stable_reference( - code_object_pickler.get_code_path( - before_module_with_functions.my_function()).replace( - "before_module_with_functions", - "after_module_add_lambda_variable")), - after_module_add_lambda_variable.my_function().__code__, - ) - - def test_get_code_remove_lambda_variable_in_function_raises_exception(self): - with self.assertRaisesRegex(AttributeError, "object has no attribute"): - code_object_pickler._get_code_from_stable_reference( - code_object_pickler.get_code_path( - before_module_with_lambdas.my_function()).replace( - "before_module_with_lambdas", - "after_module_remove_lambda_variable")) - - def test_identifiers_new_local_variable_in_class_paths_match(self): - self.assertEqual( - code_object_pickler.get_code_path( - before_module_with_classes.AddLocalVariable.my_method( - self)).replace("before_module_with_classes", "module_name"), - code_object_pickler.get_code_path( - after_module_with_classes.AddLocalVariable.my_method(self)).replace( - "after_module_with_classes", "module_name"), - ) - - def test_identifiers_remove_local_variable_in_class_paths_match(self): - self.assertEqual( - code_object_pickler.get_code_path( - before_module_with_classes.RemoveLocalVariable.my_method( - self)).replace("before_module_with_classes", "module_name"), - code_object_pickler.get_code_path( - after_module_with_classes.RemoveLocalVariable.my_method( - self)).replace("after_module_with_classes", "module_name"), - ) - - def test_identifiers_add_lambda_variable_in_class_paths_match(self): - self.assertNotEqual( - code_object_pickler.get_code_path( - before_module_with_classes.AddLambdaVariable.my_method( - self)).replace("before_module_with_classes", "module_name"), - code_object_pickler.get_code_path( - after_module_with_classes.AddLambdaVariable.my_method( - self)).replace("after_module_with_classes", "module_name"), - ) - - def test_identifiers_remove_lambda_variable_in_class_paths_match(self): - self.assertNotEqual( - code_object_pickler.get_code_path( - before_module_with_classes.RemoveLambdaVariable.my_method( - self)).replace("before_module_with_classes", "module_name"), - code_object_pickler.get_code_path( - after_module_with_classes.RemoveLambdaVariable.my_method( - self)).replace("after_module_with_classes", "module_name"), - ) - - def test_identifiers_nested_function_in_class_paths_match(self): - self.assertEqual( - code_object_pickler.get_code_path( - before_module_with_classes.ClassWithNestedFunction.my_method( - self)).replace("before_module_with_classes", "module_name"), - code_object_pickler.get_code_path( - after_module_with_classes.ClassWithNestedFunction.my_method( - self)).replace("after_module_with_classes", "module_name"), - ) - - def test_identifiers_nested_function_2_in_class_paths_match(self): - self.assertEqual( - code_object_pickler.get_code_path( - before_module_with_classes.ClassWithNestedFunction2.my_method( - self)).replace("before_module_with_classes", "module_name"), - code_object_pickler.get_code_path( - after_module_with_classes.ClassWithNestedFunction2.my_method( - self)).replace("after_module_with_classes", "module_name"), - ) - - def test_identifiers_add_new_function_in_class_paths_match(self): - self.assertEqual( - code_object_pickler.get_code_path( - before_module_with_classes.ClassWithTwoMethods.my_method( - self)).replace("before_module_with_classes", "module_name"), - code_object_pickler.get_code_path( - after_module_with_classes.ClassWithTwoMethods.my_method( - self)).replace("after_module_with_classes", "module_name"), - ) - - def test_identifiers_remove_function_in_class_paths_match(self): - self.assertEqual( - code_object_pickler.get_code_path( - before_module_with_classes.RemoveMethod.my_method(self)).replace( - "before_module_with_classes", "module_name"), - code_object_pickler.get_code_path( - after_module_with_classes.RemoveMethod.my_method(self)).replace( - "after_module_with_classes", "module_name"), - ) - - def test_identifiers_add_global_variable_paths_match(self): - self.assertEqual( - code_object_pickler.get_code_path( - before_module_with_functions.my_function()).replace( - "before_module_with_functions", "module_name"), - code_object_pickler.get_code_path( - after_module_with_global_variable.my_function()).replace( - "after_module_with_global_variable", "module_name"), - ) - - def test_identifiers_add_top_level_function_paths_match(self): - self.assertEqual( - code_object_pickler.get_code_path( - before_module_with_functions.my_function()).replace( - "before_module_with_functions", "module_name"), - code_object_pickler.get_code_path( - after_module_add_function.my_function()).replace( - "after_module_add_function", "module_name"), - ) - - def test_identifiers_add_local_variable_in_function_paths_match(self): - self.assertEqual( - code_object_pickler.get_code_path( - before_module_with_functions.my_function()).replace( - "before_module_with_functions", "module_name"), - code_object_pickler.get_code_path( - after_module_add_variable.my_function()).replace( - "after_module_add_variable", "module_name"), - ) - - def test_identifiers_remove_local_variable_in_function_paths_match(self): - self.assertEqual( - code_object_pickler.get_code_path( - before_module_with_functions.my_function()).replace( - "before_module_with_functions", "module_name"), - code_object_pickler.get_code_path( - after_module_remove_variable.my_function()).replace( - "after_module_remove_variable", "module_name"), - ) - - def test_identifiers_nested_function_in_function_paths_match(self): - self.assertEqual( - code_object_pickler.get_code_path( - before_module_with_functions.my_function()).replace( - "before_module_with_functions", "module_name"), - code_object_pickler.get_code_path( - after_module_with_nested_function.my_function()).replace( - "after_module_with_nested_function", "module_name"), - ) - - def test_identifiers_nested_function_2_in_function_paths_match(self): - self.assertEqual( - code_object_pickler.get_code_path( - before_module_with_functions.my_function()).replace( - "before_module_with_functions", "module_name"), - code_object_pickler.get_code_path( - after_module_with_nested_function_2.my_function()).replace( - "after_module_with_nested_function_2", "module_name"), - ) - - def test_identifiers_add_class_paths_match(self): - self.assertEqual( - code_object_pickler.get_code_path( - before_module_with_functions.my_function()).replace( - "before_module_with_functions", "module_name"), - code_object_pickler.get_code_path( - after_module_with_single_class.my_function()).replace( - "after_module_with_single_class", "module_name"), - ) - - def test_identifiers_add_lambda_variable_in_function_no_match(self): - self.assertNotEqual( - code_object_pickler.get_code_path( - before_module_with_functions.my_function()).replace( - "before_module_with_functions", "module_name"), - code_object_pickler.get_code_path( - after_module_add_lambda_variable.my_function()).replace( - "after_module_add_lambda_variable", "module_name"), - ) - - def test_identifiers_remove_lambda_variable_in_function_no_match(self): - self.assertNotEqual( - code_object_pickler.get_code_path( - before_module_with_lambdas.my_function()).replace( - "before_module_with_lambdas", "module_name"), - code_object_pickler.get_code_path( - after_module_remove_lambda_variable.my_function()).replace( - "after_module_remove_lambda_variable", "module_name"), - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/sdks/python/apache_beam/internal/test_cases/__init__.py b/sdks/python/apache_beam/internal/test_data/__init__.py similarity index 89% rename from sdks/python/apache_beam/internal/test_cases/__init__.py rename to sdks/python/apache_beam/internal/test_data/__init__.py index 0bce5d68f724..aee556c15717 100644 --- a/sdks/python/apache_beam/internal/test_cases/__init__.py +++ b/sdks/python/apache_beam/internal/test_data/__init__.py @@ -15,4 +15,4 @@ # limitations under the License. # -"""For internal use only; no backwards-compatibility guarantees.""" +"""Test data to validate that code identifiers are invariant to small modifications.""" diff --git a/sdks/python/apache_beam/internal/test_cases/before_module_with_functions.py b/sdks/python/apache_beam/internal/test_data/module_1.py similarity index 100% rename from sdks/python/apache_beam/internal/test_cases/before_module_with_functions.py rename to sdks/python/apache_beam/internal/test_data/module_1.py diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_single_class.py b/sdks/python/apache_beam/internal/test_data/module_1_class_added.py similarity index 100% rename from sdks/python/apache_beam/internal/test_cases/after_module_with_single_class.py rename to sdks/python/apache_beam/internal/test_data/module_1_class_added.py diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_add_function.py b/sdks/python/apache_beam/internal/test_data/module_1_function_added.py similarity index 100% rename from sdks/python/apache_beam/internal/test_cases/after_module_add_function.py rename to sdks/python/apache_beam/internal/test_data/module_1_function_added.py diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_global_variable.py b/sdks/python/apache_beam/internal/test_data/module_1_global_variable_added.py similarity index 100% rename from sdks/python/apache_beam/internal/test_cases/after_module_with_global_variable.py rename to sdks/python/apache_beam/internal/test_data/module_1_global_variable_added.py diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_add_lambda_variable.py b/sdks/python/apache_beam/internal/test_data/module_1_lambda_variable_added.py similarity index 100% rename from sdks/python/apache_beam/internal/test_cases/after_module_add_lambda_variable.py rename to sdks/python/apache_beam/internal/test_data/module_1_lambda_variable_added.py diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_add_variable.py b/sdks/python/apache_beam/internal/test_data/module_1_local_variable_added.py similarity index 100% rename from sdks/python/apache_beam/internal/test_cases/after_module_add_variable.py rename to sdks/python/apache_beam/internal/test_data/module_1_local_variable_added.py diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_remove_variable.py b/sdks/python/apache_beam/internal/test_data/module_1_local_variable_removed.py similarity index 100% rename from sdks/python/apache_beam/internal/test_cases/after_module_remove_variable.py rename to sdks/python/apache_beam/internal/test_data/module_1_local_variable_removed.py diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py b/sdks/python/apache_beam/internal/test_data/module_1_nested_function_2_added.py similarity index 100% rename from sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function.py rename to sdks/python/apache_beam/internal/test_data/module_1_nested_function_2_added.py diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function_2.py b/sdks/python/apache_beam/internal/test_data/module_1_nested_function_added.py similarity index 100% rename from sdks/python/apache_beam/internal/test_cases/after_module_with_nested_function_2.py rename to sdks/python/apache_beam/internal/test_data/module_1_nested_function_added.py diff --git a/sdks/python/apache_beam/internal/test_cases/before_module_with_classes.py b/sdks/python/apache_beam/internal/test_data/module_2.py similarity index 100% rename from sdks/python/apache_beam/internal/test_cases/before_module_with_classes.py rename to sdks/python/apache_beam/internal/test_data/module_2.py diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py b/sdks/python/apache_beam/internal/test_data/module_2_modified.py similarity index 100% rename from sdks/python/apache_beam/internal/test_cases/after_module_with_classes.py rename to sdks/python/apache_beam/internal/test_data/module_2_modified.py diff --git a/sdks/python/apache_beam/internal/test_cases/before_module_with_lambdas.py b/sdks/python/apache_beam/internal/test_data/module_3.py similarity index 100% rename from sdks/python/apache_beam/internal/test_cases/before_module_with_lambdas.py rename to sdks/python/apache_beam/internal/test_data/module_3.py diff --git a/sdks/python/apache_beam/internal/test_cases/after_module_remove_lambda_variable.py b/sdks/python/apache_beam/internal/test_data/module_3_modified.py similarity index 100% rename from sdks/python/apache_beam/internal/test_cases/after_module_remove_lambda_variable.py rename to sdks/python/apache_beam/internal/test_data/module_3_modified.py diff --git a/sdks/python/apache_beam/internal/test_cases/module_with_default_argument.py b/sdks/python/apache_beam/internal/test_data/module_with_default_argument.py similarity index 100% rename from sdks/python/apache_beam/internal/test_cases/module_with_default_argument.py rename to sdks/python/apache_beam/internal/test_data/module_with_default_argument.py From 2a14db51b6c1133d4e7ff46cdf9de09a483553e9 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Thu, 14 Aug 2025 12:07:28 -0400 Subject: [PATCH 138/142] fix error --- sdks/python/apache_beam/internal/code_object_pickler_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index 5239dac87040..7f7eb1da085a 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -292,7 +292,7 @@ def test_adding_lambda_variable_in_class_preserves_object(self): def test_removing_lambda_variable_in_class_changes_object(self): with self.assertRaisesRegex(AttributeError, "object has no attribute"): - code_object_pickler.( + code_object_pickler.get_code_from_stable_reference( code_object_pickler.get_code_path( module_2.RemoveLambdaVariable.my_method( self)).replace( From 81f5a337ae911b3817f5eac2aa50ee68c3c9363a Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Thu, 14 Aug 2025 14:07:57 -0400 Subject: [PATCH 139/142] rename functions in code_object_pickler.py --- .../internal/code_object_pickler.py | 23 +-- .../internal/code_object_pickler_test.py | 162 +++++++++--------- 2 files changed, 93 insertions(+), 92 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler.py b/sdks/python/apache_beam/internal/code_object_pickler.py index 397bf8f1079c..1d9c12547abf 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler.py +++ b/sdks/python/apache_beam/internal/code_object_pickler.py @@ -24,7 +24,7 @@ A code object identifier is a unique identifier for a code object that provides a unique reference to the code object in the context where the code is defined -and is invariant to small changes in the currounding code. +and is invariant to small changes in the surrounding code. The code object identifiers consists of a sequence of the following parts separated by periods: @@ -64,7 +64,7 @@ def get_normalized_path(path): return path -def get_code_path(callable: types.FunctionType): +def get_code_object_identifier(callable: types.FunctionType): """Returns the code object identifier for a given callable. Args: @@ -411,11 +411,11 @@ def _get_code_object_from_lambda_with_hash_pattern( raise AttributeError(f'Could not find code object with path: {path}') -def get_code_from_stable_reference(path: str): - """Returns the code object from a stable reference code object identifier. +def get_code_from_identifier(code_object_identifier: str): + """Returns the code object corresponding to thefrom a code object identifier. Args: - path: A string representing the code object identifier. + code_object_identifier: A string representing the code object identifier. Returns: The code object. @@ -424,25 +424,26 @@ def get_code_from_stable_reference(path: str): ValueError: If the path is empty or invalid. AttributeError: If the attribute is not found. """ - if not path: + if not code_object_identifier: raise ValueError('Path must not be empty.') - parts = path.split('.') + parts = code_object_identifier.split('.') obj = sys.modules[parts[0]] for part in parts[1:]: if name_result := _SINGLE_NAME_PATTERN.fullmatch(part): - obj = _get_code_object_from_single_name_pattern(obj, name_result, path) + obj = _get_code_object_from_single_name_pattern( + obj, name_result, code_object_identifier) elif lambda_with_args_result := _LAMBDA_WITH_ARGS_PATTERN.fullmatch(part): obj = _get_code_object_from_lambda_with_args_pattern( - obj, lambda_with_args_result, path) + obj, lambda_with_args_result, code_object_identifier) elif lambda_with_hash_result := _LAMBDA_WITH_HASH_PATTERN.fullmatch(part): obj = _get_code_object_from_lambda_with_hash_pattern( - obj, lambda_with_hash_result, path) + obj, lambda_with_hash_result, code_object_identifier) elif default_result := _DEFAULT_PATTERN.fullmatch(part): index = int(default_result.group(2)) if index >= len(obj.__defaults__): raise ValueError( f'Index {index} is out of bounds for obj.__defaults__' - f' {len(obj.__defaults__)} in path {path}') + f' {len(obj.__defaults__)} in path {code_object_identifier}') obj = getattr(obj, '__defaults__')[index] else: obj = getattr(obj, part) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index 7f7eb1da085a..50bbb84ea59d 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -207,60 +207,60 @@ def get_lambda_from_dictionary(): ] -class CodePathGenerationTest(unittest.TestCase): +class CodeObjectIdentifierGenerationTest(unittest.TestCase): @parameterized.expand(test_cases) - def test_get_code_path(self, callable, expected_path): - actual = code_object_pickler.get_code_path(callable) + def test_get_code_object_identifier(self, callable, expected_path): + actual = code_object_pickler.code_object_identifier(callable) self.assertEqual(actual, expected) @parameterized.expand(test_cases) - def test_get_code_from_stable_reference(self, expected_callable, path): - actual = code_object_pickler.get_code_from_stable_reference(path) + def test_get_code_from_identifier(self, expected_callable, path): + actual = code_object_pickler.get_code_from_identifier(path) self.assertEqual(actual, callable.__code__) @parameterized.expand(test_cases) def test_roundtrip(self, callable, unused_path): - path = code_object_pickler.get_code_path(callable) - actual = code_object_pickler.get_code_from_stable_reference(path) + path = code_object_pickler.code_object_identifier(callable) + actual = code_object_pickler.get_code_from_identifier(path) self.assertEqual(actual, callable.__code__) -class GetCodeFromStableReferenceTest(unittest.TestCase): +class GetCodeFromCodeObjectIdentifierTest(unittest.TestCase): def empty_path_raises_exception(self): with self.assertRaisesRegex(ValueError, "Path must not be empty"): - code_object_pickler.get_code_from_stable_reference("") + code_object_pickler.test_get_code_from_identifier("") def invalid_default_index_raises_exception(self): with self.assertRaisesRegex(ValueError, "out of bounds"): - code_object_pickler.get_code_from_stable_reference( + code_object_pickler.test_get_code_from_identifier( "apache_beam.internal.test_cases.module_with_default_argument." "function_with_lambda_default_argument.__defaults__[1]") def invalid_single_name_path_raises_exception(self): with self.assertRaisesRegex(AttributeError, "Could not find code object with path"): - code_object_pickler.get_code_from_stable_reference( + code_object_pickler.get_code_from_identifier( "apache_beam.internal.test_cases.module_3." "my_function.__code__.co_consts[something]") def invalid_lambda_with_args_path_raises_exception(self): with self.assertRaisesRegex(AttributeError, "Could not find code object with path"): - code_object_pickler.get_code_from_stable_reference( + code_object_pickler.get_code_from_identifier( "apache_beam.internal.test_cases.module_3." "my_function.__code__.co_consts[, ('x',)]") def invalid_lambda_with_hash_path_raises_exception(self): with self.assertRaisesRegex(AttributeError, "Could not find code object with path"): - code_object_pickler.get_code_from_stable_reference( + code_object_pickler.get_code_from_identifier( "apache_beam.internal.test_cases.module_3." "my_function.__code__.co_consts[, ('',), 1234567890]") def test_adding_local_variable_in_class_preserves_object(self): self.assertEqual( - code_object_pickler.get_code_from_stable_reference( - code_object_pickler.get_code_path( + code_object_pickler.get_code_from_identifier( + code_object_pickler.code_object_identifier( module_2.AddLocalVariable.my_method( self)).replace( "module_2", @@ -270,8 +270,8 @@ def test_adding_local_variable_in_class_preserves_object(self): def test_removing_local_variable_in_class_preserves_object(self): self.assertEqual( - code_object_pickler.get_code_from_stable_reference( - code_object_pickler.get_code_path( + code_object_pickler.get_code_from_identifier( + code_object_pickler.code_object_identifier( module_2.RemoveLocalVariable.my_method( self)).replace( "module_2", @@ -281,8 +281,8 @@ def test_removing_local_variable_in_class_preserves_object(self): def test_adding_lambda_variable_in_class_preserves_object(self): self.assertEqual( - code_object_pickler.get_code_from_stable_reference( - code_object_pickler.get_code_path( + code_object_pickler.get_code_from_identifier( + code_object_pickler.code_object_identifier( module_2.AddLambdaVariable.my_method( self)).replace( "module_2", @@ -292,8 +292,8 @@ def test_adding_lambda_variable_in_class_preserves_object(self): def test_removing_lambda_variable_in_class_changes_object(self): with self.assertRaisesRegex(AttributeError, "object has no attribute"): - code_object_pickler.get_code_from_stable_reference( - code_object_pickler.get_code_path( + code_object_pickler.get_code_from_identifier( + code_object_pickler.code_object_identifier( module_2.RemoveLambdaVariable.my_method( self)).replace( "module_2", @@ -301,8 +301,8 @@ def test_removing_lambda_variable_in_class_changes_object(self): def test_adding_nested_function_in_class_preserves_object(self): self.assertEqual( - code_object_pickler.get_code_from_stable_reference( - code_object_pickler.get_code_path( + code_object_pickler.get_code_from_identifier( + code_object_pickler.code_object_identifier( module_2.ClassWithNestedFunction.my_method( self)).replace( "module_2", @@ -313,8 +313,8 @@ def test_adding_nested_function_in_class_preserves_object(self): def test_adding_nested_function_2_in_class_preserves_object(self): self.assertEqual( - code_object_pickler.get_code_from_stable_reference( - code_object_pickler.get_code_path( + code_object_pickler.get_code_from_identifier( + code_object_pickler.code_object_identifier( module_2.ClassWithNestedFunction2.my_method( self)).replace( "module_2", @@ -325,8 +325,8 @@ def test_adding_nested_function_2_in_class_preserves_object(self): def test_adding_new_function_in_class_preserves_object(self): self.assertEqual( - code_object_pickler.get_code_from_stable_reference( - code_object_pickler.get_code_path( + code_object_pickler.get_code_from_identifier( + code_object_pickler.code_object_identifier( module_2.ClassWithTwoMethods.my_method( self)).replace( "module_2", @@ -336,8 +336,8 @@ def test_adding_new_function_in_class_preserves_object(self): def test_removing_method_in_class_preserves_object(self): self.assertEqual( - code_object_pickler.get_code_from_stable_reference( - code_object_pickler.get_code_path( + code_object_pickler.get_code_from_identifier( + code_object_pickler.code_object_identifier( module_2.RemoveMethod.my_method( self)).replace( "module_2", @@ -347,8 +347,8 @@ def test_removing_method_in_class_preserves_object(self): def test_adding_global_variable_preserves_object(self): self.assertEqual( - code_object_pickler.get_code_from_stable_reference( - code_object_pickler.get_code_path( + code_object_pickler.get_code_from_identifier( + code_object_pickler.code_object_identifier( module_1.my_function()).replace( "module_1", "module_1_global_variable_added", @@ -358,8 +358,8 @@ def test_adding_global_variable_preserves_object(self): def test_adding_top_level_function_preserves_object(self): self.assertEqual( - code_object_pickler.get_code_from_stable_reference( - code_object_pickler.get_code_path( + code_object_pickler.get_code_from_identifier( + code_object_pickler.code_object_identifier( module_1.my_function()).replace( "module_1", "module_1_function_added")), @@ -368,8 +368,8 @@ def test_adding_top_level_function_preserves_object(self): def test_adding_local_variable_in_function_preserves_object(self): self.assertEqual( - code_object_pickler.get_code_from_stable_reference( - code_object_pickler.get_code_path( + code_object_pickler.get_code_from_identifier( + code_object_pickler.code_object_identifier( module_1.my_function()).replace( "module_1", "module_1_local_variable_added")), @@ -378,8 +378,8 @@ def test_adding_local_variable_in_function_preserves_object(self): def test_removing_local_variable_in_function_preserves_object(self): self.assertEqual( - code_object_pickler.get_code_from_stable_reference( - code_object_pickler.get_code_path( + code_object_pickler.get_code_from_identifier( + code_object_pickler.code_object_identifier( module_1.my_function()).replace( "module_1", "module_1_local_variable_removed")), @@ -388,8 +388,8 @@ def test_removing_local_variable_in_function_preserves_object(self): def test_adding_nested_function_in_function_preserves_object(self): self.assertEqual( - code_object_pickler.get_code_from_stable_reference( - code_object_pickler.get_code_path( + code_object_pickler.get_code_from_identifier( + code_object_pickler.code_object_identifier( module_1.my_function()).replace( "module_1", "module_1_nested_function_added")), @@ -398,8 +398,8 @@ def test_adding_nested_function_in_function_preserves_object(self): def test_adding_nested_function_2_in_function_preserves_object(self): self.assertEqual( - code_object_pickler.get_code_from_stable_reference( - code_object_pickler.get_code_path( + code_object_pickler.get_code_from_identifier( + code_object_pickler.code_object_identifier( module_1.my_function()).replace( "module_1", "module_1_nested_function_2_added")), @@ -408,8 +408,8 @@ def test_adding_nested_function_2_in_function_preserves_object(self): def test_adding_class_preserves_object(self): self.assertEqual( - code_object_pickler.get_code_from_stable_reference( - code_object_pickler.get_code_path( + code_object_pickler.get_code_from_identifier( + code_object_pickler.code_object_identifier( module_1.my_function()).replace( "module_1", "module_1_class_added")), @@ -418,8 +418,8 @@ def test_adding_class_preserves_object(self): def test_adding_lambda_variable_in_function_preserves_object(self): self.assertEqual( - code_object_pickler.get_code_from_stable_reference( - code_object_pickler.get_code_path( + code_object_pickler.get_code_from_identifier( + code_object_pickler.code_object_identifier( module_1.my_function()).replace( "module_1", "module_1_lambda_variable_added")), @@ -428,8 +428,8 @@ def test_adding_lambda_variable_in_function_preserves_object(self): def test_removing_lambda_variable_in_function_raises_exception(self): with self.assertRaisesRegex(AttributeError, "object has no attribute"): - code_object_pickler.get_code_from_stable_reference( - code_object_pickler.get_code_path( + code_object_pickler.get_code_from_identifier( + code_object_pickler.code_object_identifier( module_3.my_function()).replace( "module_3", "module_3_modified")) @@ -438,170 +438,170 @@ def test_removing_lambda_variable_in_function_raises_exception(self): class CodePathStabilityTest(unittest.TestCase): def test_adding_local_variable_in_class_preserves_path(self): self.assertEqual( - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_2.AddLocalVariable.my_method( self)).replace("module_2", "module_name"), - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_2_modified.AddLocalVariable.my_method(self)).replace( "module_2_modified", "module_name"), ) def test_removing_local_variable_in_class_preserves_path(self): self.assertEqual( - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_2.RemoveLocalVariable.my_method( self)).replace("module_2", "module_name"), - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_2_modified.RemoveLocalVariable.my_method( self)).replace("module_2_modified", "module_name"), ) def test_adding_lambda_variable_in_class_changes_path(self): self.assertNotEqual( - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_2.AddLambdaVariable.my_method( self)).replace("module_2", "module_name"), - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_2_modified.AddLambdaVariable.my_method( self)).replace("module_2_modified", "module_name"), ) def test_removing_lambda_variable_in_class_changes_path(self): self.assertNotEqual( - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_2.RemoveLambdaVariable.my_method( self)).replace("module_2", "module_name"), - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_2_modified.RemoveLambdaVariable.my_method( self)).replace("module_2_modified", "module_name"), ) def test_adding_nested_function_in_class_preserves_path(self): self.assertEqual( - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_2.ClassWithNestedFunction.my_method( self)).replace("module_2", "module_name"), - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_2_modified.ClassWithNestedFunction.my_method( self)).replace("module_2_modified", "module_name"), ) def test_adding_nested_function_2_in_class_preserves_path(self): self.assertEqual( - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_2.ClassWithNestedFunction2.my_method( self)).replace("module_2", "module_name"), - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_2_modified.ClassWithNestedFunction2.my_method( self)).replace("module_2_modified", "module_name"), ) def test_adding_new_function_in_class_preserves_path(self): self.assertEqual( - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_2.ClassWithTwoMethods.my_method( self)).replace("module_2", "module_name"), - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_2_modified.ClassWithTwoMethods.my_method( self)).replace("module_2_modified", "module_name"), ) def test_removing_function_in_class_preserves_path(self): self.assertEqual( - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_2.RemoveMethod.my_method(self)).replace( "module_2", "module_name"), - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_2_modified.RemoveMethod.my_method(self)).replace( "module_2_modified", "module_name"), ) def test_adding_global_variable_preserves_path(self): self.assertEqual( - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_1.my_function()).replace( "module_1", "module_name"), - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_1_global_variable_added.my_function()).replace( "module_1_global_variable_added", "module_name"), ) def test_adding_top_level_function_preserves_path(self): self.assertEqual( - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_1.my_function()).replace( "module_1", "module_name"), - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_1_function_added.my_function()).replace( "module_1_function_added", "module_name"), ) def test_adding_local_variable_in_function_preserves_path(self): self.assertEqual( - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_1.my_function()).replace( "module_1", "module_name"), - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_1_local_variable_added.my_function()).replace( "module_1_local_variable_added", "module_name"), ) def test_removing_local_variable_in_function_preserves_path(self): self.assertEqual( - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_1.my_function()).replace( "module_1", "module_name"), - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_1_local_variable_removed.my_function()).replace( "module_1_local_variable_removed", "module_name"), ) def test_adding_nested_function_in_function_preserves_path(self): self.assertEqual( - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_1.my_function()).replace( "module_1", "module_name"), - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_1_nested_function_added.my_function()).replace( "module_1_nested_function_added", "module_name"), ) def test_adding_nested_function_2_in_function_preserves_path(self): self.assertEqual( - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_1.my_function()).replace( "module_1", "module_name"), - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_1_nested_function_2_added.my_function()).replace( "module_1_nested_function_2_added", "module_name"), ) def test_adding_class_preserves_path(self): self.assertEqual( - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_1.my_function()).replace( "module_1", "module_name"), - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_1_class_added.my_function()).replace( "module_1_class_added", "module_name"), ) def test_adding_lambda_variable_in_function_changes_path(self): self.assertNotEqual( - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_1.my_function()).replace( "module_1", "module_name"), - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_1_lambda_variable_added.my_function()).replace( "module_1_lambda_variable_added", "module_name"), ) def test_removing_lambda_variable_in_function_changes_path(self): self.assertNotEqual( - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_3.my_function()).replace( "module_3", "module_name"), - code_object_pickler.get_code_path( + code_object_pickler.code_object_identifier( module_3_modified.my_function()).replace( "module_3_modified", "module_name"), ) From a17c69dc50804133b810008587b6c5e40d53e13b Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Thu, 14 Aug 2025 16:27:47 -0400 Subject: [PATCH 140/142] fix errors --- .../internal/code_object_pickler.py | 8 +- .../internal/code_object_pickler_test.py | 263 ++++++++---------- .../internal/test_data/__init__.py | 4 +- 3 files changed, 121 insertions(+), 154 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler.py b/sdks/python/apache_beam/internal/code_object_pickler.py index 1d9c12547abf..9665558476f0 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler.py +++ b/sdks/python/apache_beam/internal/code_object_pickler.py @@ -105,11 +105,11 @@ def _extend_path(prefix: str, current_path: Optional[str]): Returns: The extended path. """ - if suffix is None: + if current_path is None: return None - if not suffix: + if not current_path: return prefix - return prefix + '.' + suffix + return prefix + '.' + current_path def _search( @@ -431,7 +431,7 @@ def get_code_from_identifier(code_object_identifier: str): for part in parts[1:]: if name_result := _SINGLE_NAME_PATTERN.fullmatch(part): obj = _get_code_object_from_single_name_pattern( - obj, name_result, code_object_identifier) + obj, name_result, code_object_identifier) elif lambda_with_args_result := _LAMBDA_WITH_ARGS_PATTERN.fullmatch(part): obj = _get_code_object_from_lambda_with_args_pattern( obj, lambda_with_args_result, code_object_identifier) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index 50bbb84ea59d..f35d94b0dfc2 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -24,7 +24,6 @@ # pylint: disable=unused-import from apache_beam.internal import code_object_pickler -from apache_beam.internal import code_object_pickler from apache_beam.internal.test_data import module_1_class_added from apache_beam.internal.test_data import module_1_function_added from apache_beam.internal.test_data import module_1_global_variable_added @@ -210,17 +209,17 @@ def get_lambda_from_dictionary(): class CodeObjectIdentifierGenerationTest(unittest.TestCase): @parameterized.expand(test_cases) def test_get_code_object_identifier(self, callable, expected_path): - actual = code_object_pickler.code_object_identifier(callable) - self.assertEqual(actual, expected) + actual = code_object_pickler.get_code_object_identifier(callable) + self.assertEqual(actual, expected_path) @parameterized.expand(test_cases) def test_get_code_from_identifier(self, expected_callable, path): actual = code_object_pickler.get_code_from_identifier(path) - self.assertEqual(actual, callable.__code__) + self.assertEqual(actual, expected_callable.__code__) @parameterized.expand(test_cases) def test_roundtrip(self, callable, unused_path): - path = code_object_pickler.code_object_identifier(callable) + path = code_object_pickler.get_code_object_identifier(callable) actual = code_object_pickler.get_code_from_identifier(path) self.assertEqual(actual, callable.__code__) @@ -260,53 +259,43 @@ def invalid_lambda_with_hash_path_raises_exception(self): def test_adding_local_variable_in_class_preserves_object(self): self.assertEqual( code_object_pickler.get_code_from_identifier( - code_object_pickler.code_object_identifier( - module_2.AddLocalVariable.my_method( - self)).replace( - "module_2", - "module_2_modified")), + code_object_pickler.get_code_object_identifier( + module_2.AddLocalVariable.my_method(self)).replace( + "module_2", "module_2_modified")), module_2_modified.AddLocalVariable.my_method(self).__code__, ) def test_removing_local_variable_in_class_preserves_object(self): self.assertEqual( code_object_pickler.get_code_from_identifier( - code_object_pickler.code_object_identifier( - module_2.RemoveLocalVariable.my_method( - self)).replace( - "module_2", - "module_2_modified")), + code_object_pickler.get_code_object_identifier( + module_2.RemoveLocalVariable.my_method(self)).replace( + "module_2", "module_2_modified")), module_2_modified.RemoveLocalVariable.my_method(self).__code__, ) def test_adding_lambda_variable_in_class_preserves_object(self): self.assertEqual( code_object_pickler.get_code_from_identifier( - code_object_pickler.code_object_identifier( - module_2.AddLambdaVariable.my_method( - self)).replace( - "module_2", - "module_2_modified")), + code_object_pickler.get_code_object_identifier( + module_2.AddLambdaVariable.my_method(self)).replace( + "module_2", "module_2_modified")), module_2_modified.AddLambdaVariable.my_method(self).__code__, ) def test_removing_lambda_variable_in_class_changes_object(self): with self.assertRaisesRegex(AttributeError, "object has no attribute"): code_object_pickler.get_code_from_identifier( - code_object_pickler.code_object_identifier( - module_2.RemoveLambdaVariable.my_method( - self)).replace( - "module_2", - "module_2_modified")) + code_object_pickler.get_code_object_identifier( + module_2.RemoveLambdaVariable.my_method(self)).replace( + "module_2", "module_2_modified")) def test_adding_nested_function_in_class_preserves_object(self): self.assertEqual( code_object_pickler.get_code_from_identifier( - code_object_pickler.code_object_identifier( - module_2.ClassWithNestedFunction.my_method( - self)).replace( - "module_2", - "module_2_modified")), + code_object_pickler.get_code_object_identifier( + module_2.ClassWithNestedFunction.my_method(self)).replace( + "module_2", "module_2_modified")), module_2_modified.ClassWithNestedFunction.my_method( self).__code__, ) @@ -314,11 +303,9 @@ def test_adding_nested_function_in_class_preserves_object(self): def test_adding_nested_function_2_in_class_preserves_object(self): self.assertEqual( code_object_pickler.get_code_from_identifier( - code_object_pickler.code_object_identifier( - module_2.ClassWithNestedFunction2.my_method( - self)).replace( - "module_2", - "module_2_modified")), + code_object_pickler.get_code_object_identifier( + module_2.ClassWithNestedFunction2.my_method(self)).replace( + "module_2", "module_2_modified")), module_2_modified.ClassWithNestedFunction2.my_method( self).__code__, ) @@ -326,29 +313,25 @@ def test_adding_nested_function_2_in_class_preserves_object(self): def test_adding_new_function_in_class_preserves_object(self): self.assertEqual( code_object_pickler.get_code_from_identifier( - code_object_pickler.code_object_identifier( - module_2.ClassWithTwoMethods.my_method( - self)).replace( - "module_2", - "module_2_modified")), + code_object_pickler.get_code_object_identifier( + module_2.ClassWithTwoMethods.my_method(self)).replace( + "module_2", "module_2_modified")), module_2_modified.ClassWithTwoMethods.my_method(self).__code__, ) def test_removing_method_in_class_preserves_object(self): self.assertEqual( code_object_pickler.get_code_from_identifier( - code_object_pickler.code_object_identifier( - module_2.RemoveMethod.my_method( - self)).replace( - "module_2", - "module_2_modified")), + code_object_pickler.get_code_object_identifier( + module_2.RemoveMethod.my_method(self)).replace( + "module_2", "module_2_modified")), module_2_modified.RemoveMethod.my_method(self).__code__, ) def test_adding_global_variable_preserves_object(self): self.assertEqual( code_object_pickler.get_code_from_identifier( - code_object_pickler.code_object_identifier( + code_object_pickler.get_code_object_identifier( module_1.my_function()).replace( "module_1", "module_1_global_variable_added", @@ -359,249 +342,231 @@ def test_adding_global_variable_preserves_object(self): def test_adding_top_level_function_preserves_object(self): self.assertEqual( code_object_pickler.get_code_from_identifier( - code_object_pickler.code_object_identifier( + code_object_pickler.get_code_object_identifier( module_1.my_function()).replace( - "module_1", - "module_1_function_added")), + "module_1", "module_1_function_added")), module_1_function_added.my_function().__code__, ) def test_adding_local_variable_in_function_preserves_object(self): self.assertEqual( code_object_pickler.get_code_from_identifier( - code_object_pickler.code_object_identifier( + code_object_pickler.get_code_object_identifier( module_1.my_function()).replace( - "module_1", - "module_1_local_variable_added")), + "module_1", "module_1_local_variable_added")), module_1_local_variable_added.my_function().__code__, ) def test_removing_local_variable_in_function_preserves_object(self): self.assertEqual( code_object_pickler.get_code_from_identifier( - code_object_pickler.code_object_identifier( + code_object_pickler.get_code_object_identifier( module_1.my_function()).replace( - "module_1", - "module_1_local_variable_removed")), + "module_1", "module_1_local_variable_removed")), module_1_local_variable_removed.my_function().__code__, ) def test_adding_nested_function_in_function_preserves_object(self): self.assertEqual( code_object_pickler.get_code_from_identifier( - code_object_pickler.code_object_identifier( + code_object_pickler.get_code_object_identifier( module_1.my_function()).replace( - "module_1", - "module_1_nested_function_added")), + "module_1", "module_1_nested_function_added")), module_1_nested_function_added.my_function().__code__, ) def test_adding_nested_function_2_in_function_preserves_object(self): self.assertEqual( code_object_pickler.get_code_from_identifier( - code_object_pickler.code_object_identifier( + code_object_pickler.get_code_object_identifier( module_1.my_function()).replace( - "module_1", - "module_1_nested_function_2_added")), + "module_1", "module_1_nested_function_2_added")), module_1_nested_function_2_added.my_function().__code__, ) def test_adding_class_preserves_object(self): self.assertEqual( code_object_pickler.get_code_from_identifier( - code_object_pickler.code_object_identifier( + code_object_pickler.get_code_object_identifier( module_1.my_function()).replace( - "module_1", - "module_1_class_added")), + "module_1", "module_1_class_added")), module_1_class_added.my_function().__code__, ) def test_adding_lambda_variable_in_function_preserves_object(self): self.assertEqual( code_object_pickler.get_code_from_identifier( - code_object_pickler.code_object_identifier( + code_object_pickler.get_code_object_identifier( module_1.my_function()).replace( - "module_1", - "module_1_lambda_variable_added")), + "module_1", "module_1_lambda_variable_added")), module_1_lambda_variable_added.my_function().__code__, ) def test_removing_lambda_variable_in_function_raises_exception(self): with self.assertRaisesRegex(AttributeError, "object has no attribute"): code_object_pickler.get_code_from_identifier( - code_object_pickler.code_object_identifier( - module_3.my_function()).replace( - "module_3", - "module_3_modified")) + code_object_pickler.get_code_object_identifier( + module_3.my_function()).replace("module_3", "module_3_modified")) class CodePathStabilityTest(unittest.TestCase): def test_adding_local_variable_in_class_preserves_path(self): self.assertEqual( - code_object_pickler.code_object_identifier( - module_2.AddLocalVariable.my_method( - self)).replace("module_2", "module_name"), - code_object_pickler.code_object_identifier( + code_object_pickler.get_code_object_identifier( + module_2.AddLocalVariable.my_method(self)).replace( + "module_2", "module_name"), + code_object_pickler.get_code_object_identifier( module_2_modified.AddLocalVariable.my_method(self)).replace( "module_2_modified", "module_name"), ) def test_removing_local_variable_in_class_preserves_path(self): self.assertEqual( - code_object_pickler.code_object_identifier( - module_2.RemoveLocalVariable.my_method( - self)).replace("module_2", "module_name"), - code_object_pickler.code_object_identifier( - module_2_modified.RemoveLocalVariable.my_method( - self)).replace("module_2_modified", "module_name"), + code_object_pickler.get_code_object_identifier( + module_2.RemoveLocalVariable.my_method(self)).replace( + "module_2", "module_name"), + code_object_pickler.get_code_object_identifier( + module_2_modified.RemoveLocalVariable.my_method(self)).replace( + "module_2_modified", "module_name"), ) def test_adding_lambda_variable_in_class_changes_path(self): self.assertNotEqual( - code_object_pickler.code_object_identifier( - module_2.AddLambdaVariable.my_method( - self)).replace("module_2", "module_name"), - code_object_pickler.code_object_identifier( - module_2_modified.AddLambdaVariable.my_method( - self)).replace("module_2_modified", "module_name"), + code_object_pickler.get_code_object_identifier( + module_2.AddLambdaVariable.my_method(self)).replace( + "module_2", "module_name"), + code_object_pickler.get_code_object_identifier( + module_2_modified.AddLambdaVariable.my_method(self)).replace( + "module_2_modified", "module_name"), ) def test_removing_lambda_variable_in_class_changes_path(self): self.assertNotEqual( - code_object_pickler.code_object_identifier( - module_2.RemoveLambdaVariable.my_method( - self)).replace("module_2", "module_name"), - code_object_pickler.code_object_identifier( - module_2_modified.RemoveLambdaVariable.my_method( - self)).replace("module_2_modified", "module_name"), + code_object_pickler.get_code_object_identifier( + module_2.RemoveLambdaVariable.my_method(self)).replace( + "module_2", "module_name"), + code_object_pickler.get_code_object_identifier( + module_2_modified.RemoveLambdaVariable.my_method(self)).replace( + "module_2_modified", "module_name"), ) def test_adding_nested_function_in_class_preserves_path(self): self.assertEqual( - code_object_pickler.code_object_identifier( - module_2.ClassWithNestedFunction.my_method( - self)).replace("module_2", "module_name"), - code_object_pickler.code_object_identifier( - module_2_modified.ClassWithNestedFunction.my_method( - self)).replace("module_2_modified", "module_name"), + code_object_pickler.get_code_object_identifier( + module_2.ClassWithNestedFunction.my_method(self)).replace( + "module_2", "module_name"), + code_object_pickler.get_code_object_identifier( + module_2_modified.ClassWithNestedFunction.my_method(self)).replace( + "module_2_modified", "module_name"), ) def test_adding_nested_function_2_in_class_preserves_path(self): self.assertEqual( - code_object_pickler.code_object_identifier( - module_2.ClassWithNestedFunction2.my_method( - self)).replace("module_2", "module_name"), - code_object_pickler.code_object_identifier( - module_2_modified.ClassWithNestedFunction2.my_method( - self)).replace("module_2_modified", "module_name"), + code_object_pickler.get_code_object_identifier( + module_2.ClassWithNestedFunction2.my_method(self)).replace( + "module_2", "module_name"), + code_object_pickler.get_code_object_identifier( + module_2_modified.ClassWithNestedFunction2.my_method(self)).replace( + "module_2_modified", "module_name"), ) def test_adding_new_function_in_class_preserves_path(self): self.assertEqual( - code_object_pickler.code_object_identifier( - module_2.ClassWithTwoMethods.my_method( - self)).replace("module_2", "module_name"), - code_object_pickler.code_object_identifier( - module_2_modified.ClassWithTwoMethods.my_method( - self)).replace("module_2_modified", "module_name"), + code_object_pickler.get_code_object_identifier( + module_2.ClassWithTwoMethods.my_method(self)).replace( + "module_2", "module_name"), + code_object_pickler.get_code_object_identifier( + module_2_modified.ClassWithTwoMethods.my_method(self)).replace( + "module_2_modified", "module_name"), ) def test_removing_function_in_class_preserves_path(self): self.assertEqual( - code_object_pickler.code_object_identifier( + code_object_pickler.get_code_object_identifier( module_2.RemoveMethod.my_method(self)).replace( "module_2", "module_name"), - code_object_pickler.code_object_identifier( + code_object_pickler.get_code_object_identifier( module_2_modified.RemoveMethod.my_method(self)).replace( "module_2_modified", "module_name"), ) def test_adding_global_variable_preserves_path(self): self.assertEqual( - code_object_pickler.code_object_identifier( - module_1.my_function()).replace( - "module_1", "module_name"), - code_object_pickler.code_object_identifier( + code_object_pickler.get_code_object_identifier( + module_1.my_function()).replace("module_1", "module_name"), + code_object_pickler.get_code_object_identifier( module_1_global_variable_added.my_function()).replace( "module_1_global_variable_added", "module_name"), ) def test_adding_top_level_function_preserves_path(self): self.assertEqual( - code_object_pickler.code_object_identifier( - module_1.my_function()).replace( - "module_1", "module_name"), - code_object_pickler.code_object_identifier( + code_object_pickler.get_code_object_identifier( + module_1.my_function()).replace("module_1", "module_name"), + code_object_pickler.get_code_object_identifier( module_1_function_added.my_function()).replace( "module_1_function_added", "module_name"), ) def test_adding_local_variable_in_function_preserves_path(self): self.assertEqual( - code_object_pickler.code_object_identifier( - module_1.my_function()).replace( - "module_1", "module_name"), - code_object_pickler.code_object_identifier( + code_object_pickler.get_code_object_identifier( + module_1.my_function()).replace("module_1", "module_name"), + code_object_pickler.get_code_object_identifier( module_1_local_variable_added.my_function()).replace( "module_1_local_variable_added", "module_name"), ) def test_removing_local_variable_in_function_preserves_path(self): self.assertEqual( - code_object_pickler.code_object_identifier( - module_1.my_function()).replace( - "module_1", "module_name"), - code_object_pickler.code_object_identifier( + code_object_pickler.get_code_object_identifier( + module_1.my_function()).replace("module_1", "module_name"), + code_object_pickler.get_code_object_identifier( module_1_local_variable_removed.my_function()).replace( "module_1_local_variable_removed", "module_name"), ) def test_adding_nested_function_in_function_preserves_path(self): self.assertEqual( - code_object_pickler.code_object_identifier( - module_1.my_function()).replace( - "module_1", "module_name"), - code_object_pickler.code_object_identifier( + code_object_pickler.get_code_object_identifier( + module_1.my_function()).replace("module_1", "module_name"), + code_object_pickler.get_code_object_identifier( module_1_nested_function_added.my_function()).replace( "module_1_nested_function_added", "module_name"), ) def test_adding_nested_function_2_in_function_preserves_path(self): self.assertEqual( - code_object_pickler.code_object_identifier( - module_1.my_function()).replace( - "module_1", "module_name"), - code_object_pickler.code_object_identifier( + code_object_pickler.get_code_object_identifier( + module_1.my_function()).replace("module_1", "module_name"), + code_object_pickler.get_code_object_identifier( module_1_nested_function_2_added.my_function()).replace( "module_1_nested_function_2_added", "module_name"), ) def test_adding_class_preserves_path(self): self.assertEqual( - code_object_pickler.code_object_identifier( - module_1.my_function()).replace( - "module_1", "module_name"), - code_object_pickler.code_object_identifier( + code_object_pickler.get_code_object_identifier( + module_1.my_function()).replace("module_1", "module_name"), + code_object_pickler.get_code_object_identifier( module_1_class_added.my_function()).replace( "module_1_class_added", "module_name"), ) def test_adding_lambda_variable_in_function_changes_path(self): self.assertNotEqual( - code_object_pickler.code_object_identifier( - module_1.my_function()).replace( - "module_1", "module_name"), - code_object_pickler.code_object_identifier( + code_object_pickler.get_code_object_identifier( + module_1.my_function()).replace("module_1", "module_name"), + code_object_pickler.get_code_object_identifier( module_1_lambda_variable_added.my_function()).replace( "module_1_lambda_variable_added", "module_name"), ) def test_removing_lambda_variable_in_function_changes_path(self): self.assertNotEqual( - code_object_pickler.code_object_identifier( - module_3.my_function()).replace( - "module_3", "module_name"), - code_object_pickler.code_object_identifier( + code_object_pickler.get_code_object_identifier( + module_3.my_function()).replace("module_3", "module_name"), + code_object_pickler.get_code_object_identifier( module_3_modified.my_function()).replace( "module_3_modified", "module_name"), ) diff --git a/sdks/python/apache_beam/internal/test_data/__init__.py b/sdks/python/apache_beam/internal/test_data/__init__.py index aee556c15717..7f27bba88cf5 100644 --- a/sdks/python/apache_beam/internal/test_data/__init__.py +++ b/sdks/python/apache_beam/internal/test_data/__init__.py @@ -15,4 +15,6 @@ # limitations under the License. # -"""Test data to validate that code identifiers are invariant to small modifications.""" +"""Test data to validate that code identifiers are invariant to small + modifications. +""" From 7ee69f11ee695912a88e6290a4d208d134f746b8 Mon Sep 17 00:00:00 2001 From: kristynsmith Date: Thu, 14 Aug 2025 17:00:29 -0400 Subject: [PATCH 141/142] fix formatting and lint errors --- .../internal/code_object_pickler_test.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler_test.py b/sdks/python/apache_beam/internal/code_object_pickler_test.py index f35d94b0dfc2..2060533e9328 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler_test.py +++ b/sdks/python/apache_beam/internal/code_object_pickler_test.py @@ -24,6 +24,7 @@ # pylint: disable=unused-import from apache_beam.internal import code_object_pickler +from apache_beam.internal.test_data import module_1 from apache_beam.internal.test_data import module_1_class_added from apache_beam.internal.test_data import module_1_function_added from apache_beam.internal.test_data import module_1_global_variable_added @@ -32,11 +33,10 @@ from apache_beam.internal.test_data import module_1_local_variable_removed from apache_beam.internal.test_data import module_1_nested_function_2_added from apache_beam.internal.test_data import module_1_nested_function_added -from apache_beam.internal.test_data import module_1 -from apache_beam.internal.test_data import module_2_modified from apache_beam.internal.test_data import module_2 -from apache_beam.internal.test_data import module_3_modified +from apache_beam.internal.test_data import module_2_modified from apache_beam.internal.test_data import module_3 +from apache_beam.internal.test_data import module_3_modified from apache_beam.internal.test_data import module_with_default_argument @@ -296,8 +296,7 @@ def test_adding_nested_function_in_class_preserves_object(self): code_object_pickler.get_code_object_identifier( module_2.ClassWithNestedFunction.my_method(self)).replace( "module_2", "module_2_modified")), - module_2_modified.ClassWithNestedFunction.my_method( - self).__code__, + module_2_modified.ClassWithNestedFunction.my_method(self).__code__, ) def test_adding_nested_function_2_in_class_preserves_object(self): @@ -306,8 +305,7 @@ def test_adding_nested_function_2_in_class_preserves_object(self): code_object_pickler.get_code_object_identifier( module_2.ClassWithNestedFunction2.my_method(self)).replace( "module_2", "module_2_modified")), - module_2_modified.ClassWithNestedFunction2.my_method( - self).__code__, + module_2_modified.ClassWithNestedFunction2.my_method(self).__code__, ) def test_adding_new_function_in_class_preserves_object(self): @@ -571,5 +569,6 @@ def test_removing_lambda_variable_in_function_changes_path(self): "module_3_modified", "module_name"), ) + if __name__ == "__main__": unittest.main() From a9838a13709a7b1411793f8d297d03da758ae98b Mon Sep 17 00:00:00 2001 From: tvalentyn Date: Fri, 15 Aug 2025 12:00:34 +0200 Subject: [PATCH 142/142] Update sdks/python/apache_beam/internal/code_object_pickler.py --- sdks/python/apache_beam/internal/code_object_pickler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/internal/code_object_pickler.py b/sdks/python/apache_beam/internal/code_object_pickler.py index 9665558476f0..b6ea015cc06f 100644 --- a/sdks/python/apache_beam/internal/code_object_pickler.py +++ b/sdks/python/apache_beam/internal/code_object_pickler.py @@ -412,7 +412,7 @@ def _get_code_object_from_lambda_with_hash_pattern( def get_code_from_identifier(code_object_identifier: str): - """Returns the code object corresponding to thefrom a code object identifier. + """Returns the code object corresponding to the code object identifier. Args: code_object_identifier: A string representing the code object identifier.