From 44b2a6032d6edb82a79daeccec6994daa8a601e8 Mon Sep 17 00:00:00 2001 From: jab Date: Fri, 24 Aug 2018 05:08:55 +0000 Subject: [PATCH 1/4] bpo-31861: Add operator.aiter and operator.anext --- Lib/operator.py | 67 +++++++++++++++-- Lib/test/test_asyncgen.py | 75 +++++++++++++++++++ .../2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst | 1 + 3 files changed, 135 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst diff --git a/Lib/operator.py b/Lib/operator.py index fb58851fa6ef67..ed4b3719e67c36 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -10,16 +10,18 @@ This is the pure Python implementation of the module. """ -__all__ = ['abs', 'add', 'and_', 'attrgetter', 'concat', 'contains', 'countOf', - 'delitem', 'eq', 'floordiv', 'ge', 'getitem', 'gt', 'iadd', 'iand', - 'iconcat', 'ifloordiv', 'ilshift', 'imatmul', 'imod', 'imul', - 'index', 'indexOf', 'inv', 'invert', 'ior', 'ipow', 'irshift', - 'is_', 'is_not', 'isub', 'itemgetter', 'itruediv', 'ixor', 'le', - 'length_hint', 'lshift', 'lt', 'matmul', 'methodcaller', 'mod', - 'mul', 'ne', 'neg', 'not_', 'or_', 'pos', 'pow', 'rshift', - 'setitem', 'sub', 'truediv', 'truth', 'xor'] +__all__ = [ + 'abs', 'add', 'aiter', 'anext', 'and_', 'attrgetter', 'concat', 'contains', + 'countOf', 'delitem', 'eq', 'floordiv', 'ge', 'getitem', 'gt', 'iadd', + 'iand', 'iconcat', 'ifloordiv', 'ilshift', 'imatmul', 'imod', 'imul', + 'index', 'indexOf', 'inv', 'invert', 'ior', 'ipow', 'irshift', 'is_', + 'is_not', 'isub', 'itemgetter', 'itruediv', 'ixor', 'le', 'length_hint', + 'lshift', 'lt', 'matmul', 'methodcaller', 'mod', 'mul', 'ne', 'neg', 'not_', + 'or_', 'pos', 'pow', 'rshift', 'setitem', 'sub', 'truediv', 'truth', 'xor', +] from builtins import abs as _abs +from collections.abc import AsyncIterable, AsyncIterator # Comparison Operations *******************************************************# @@ -404,6 +406,55 @@ def ixor(a, b): return a +# Asynchronous Iterator Operations ********************************************# + +async def aiter(*args): + """aiter(async_iterable) -> async_iterator + aiter(async_callable, sentinel) -> async_iterator + + An async version of the iter() builtin. + """ + lenargs = len(args) + if lenargs != 1 and lenargs != 2: + raise TypeError(f'aiter expected 1 or 2 arguments, got {lenargs}') + if lenargs == 1: + obj, = args + if not isinstance(obj, AsyncIterable): + raise TypeError(f'aiter expected an AsyncIterable, got {type(obj)}') + async for i in obj.__aiter__(): + yield i + return + # lenargs == 2 + async_callable, sentinel = args + while True: + value = await async_callable() + if value == sentinel: + break + yield value + + +async def anext(*args): + """anext(async_iterator[, default]) + + Return the next item from the async iterator. + If default is given and the iterator is exhausted, + it is returned instead of raising StopAsyncIteration. + """ + lenargs = len(args) + if lenargs != 1 and lenargs != 2: + raise TypeError(f'anext expected 1 or 2 arguments, got {lenargs}') + ait = args[0] + if not isinstance(ait, AsyncIterator): + raise TypeError(f'anext expected an AsyncIterable, got {type(ait)}') + anxt = ait.__anext__ + try: + return await anxt() + except StopAsyncIteration: + if lenargs == 1: + raise + return args[1] # default + + try: from _operator import * except ImportError: diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 1f7e05b42be99c..41d243094e6a41 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -1,4 +1,5 @@ import inspect +import operator import types import unittest @@ -372,6 +373,80 @@ def tearDown(self): self.loop = None asyncio.set_event_loop_policy(None) + def test_async_gen_operator_anext(self): + async def gen(): + yield 1 + yield 2 + g = gen() + async def consume(): + results = [] + results.append(await operator.anext(g)) + results.append(await operator.anext(g)) + results.append(await operator.anext(g, 'buckle my shoe')) + return results + res = self.loop.run_until_complete(consume()) + self.assertEqual(res, [1, 2, 'buckle my shoe']) + with self.assertRaises(StopAsyncIteration): + self.loop.run_until_complete(consume()) + + def test_async_gen_operator_aiter(self): + async def gen(): + yield 1 + yield 2 + g = gen() + async def consume(): + return [i async for i in operator.aiter(g)] + res = self.loop.run_until_complete(consume()) + self.assertEqual(res, [1, 2]) + + def test_async_gen_operator_aiter_class(self): + loop = self.loop + class Gen: + async def __aiter__(self): + yield 1 + await asyncio.sleep(0.01, loop=loop) + yield 2 + g = Gen() + async def consume(): + return [i async for i in operator.aiter(g)] + res = self.loop.run_until_complete(consume()) + self.assertEqual(res, [1, 2]) + + def test_async_gen_operator_aiter_2_arg(self): + async def gen(): + yield 1 + yield 2 + yield None + g = gen() + async def foo(): + return await operator.anext(g) + async def consume(): + return [i async for i in operator.aiter(foo, None)] + res = self.loop.run_until_complete(consume()) + self.assertEqual(res, [1, 2]) + + def test_operator_anext_bad_args(self): + self._test_bad_args(operator.anext) + + def test_operator_aiter_bad_args(self): + self._test_bad_args(operator.aiter) + + def _test_bad_args(self, afn): + async def gen(): + yield 1 + async def call_with_no_args(): + await afn() + async def call_with_3_args(): + await afn(gen(), 1, 2) + async def call_with_bad_args(): + await afn(1, gen()) + with self.assertRaises(TypeError): + self.loop.run_until_complete(call_with_no_args()) + with self.assertRaises(TypeError): + self.loop.run_until_complete(call_with_3_args()) + with self.assertRaises(TypeError): + self.loop.run_until_complete(call_with_bad_args()) + async def to_list(self, gen): res = [] async for i in gen: diff --git a/Misc/NEWS.d/next/Library/2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst b/Misc/NEWS.d/next/Library/2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst new file mode 100644 index 00000000000000..f218a1aa2af71e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-08-24-01-08-09.bpo-31861.-q9RKJ.rst @@ -0,0 +1 @@ +Add the operator.aiter and operator.anext functions. Patch by Josh Bronson. From 142523bf8bb41990b9dc0970664eb45833985849 Mon Sep 17 00:00:00 2001 From: jab Date: Fri, 7 Sep 2018 12:24:20 -0400 Subject: [PATCH 2/4] address comments from first review --- Lib/operator.py | 59 +++++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/Lib/operator.py b/Lib/operator.py index ed4b3719e67c36..9a07c50af99dab 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -408,51 +408,52 @@ def ixor(a, b): # Asynchronous Iterator Operations ********************************************# -async def aiter(*args): + +_NOT_PROVIDED = object() # sentinel object to detect when a kwarg was not given + + +def aiter(obj, sentinel=_NOT_PROVIDED): """aiter(async_iterable) -> async_iterator aiter(async_callable, sentinel) -> async_iterator - An async version of the iter() builtin. + Like the iter() builtin but for async iterables and callables. """ - lenargs = len(args) - if lenargs != 1 and lenargs != 2: - raise TypeError(f'aiter expected 1 or 2 arguments, got {lenargs}') - if lenargs == 1: - obj, = args + if sentinel is _NOT_PROVIDED: if not isinstance(obj, AsyncIterable): raise TypeError(f'aiter expected an AsyncIterable, got {type(obj)}') - async for i in obj.__aiter__(): - yield i - return - # lenargs == 2 - async_callable, sentinel = args - while True: - value = await async_callable() - if value == sentinel: - break - yield value - - -async def anext(*args): + if isinstance(obj, AsyncIterator): + return obj + return (i async for i in obj) + + if not callable(obj): + raise TypeError(f'aiter expected an async callable, got {type(obj)}') + + async def ait(): + while True: + value = await obj() + if value == sentinel: + break + yield value + + return ait() + + +async def anext(async_iterator, default=_NOT_PROVIDED): """anext(async_iterator[, default]) Return the next item from the async iterator. If default is given and the iterator is exhausted, it is returned instead of raising StopAsyncIteration. """ - lenargs = len(args) - if lenargs != 1 and lenargs != 2: - raise TypeError(f'anext expected 1 or 2 arguments, got {lenargs}') - ait = args[0] - if not isinstance(ait, AsyncIterator): - raise TypeError(f'anext expected an AsyncIterable, got {type(ait)}') - anxt = ait.__anext__ + if not isinstance(async_iterator, AsyncIterator): + raise TypeError(f'anext expected an AsyncIterator, got {type(async_iterator)}') + anxt = async_iterator.__anext__ try: return await anxt() except StopAsyncIteration: - if lenargs == 1: + if default is _NOT_PROVIDED: raise - return args[1] # default + return default try: From 3ebce05681c0cd6095cc8efb45f0ec024a05a48b Mon Sep 17 00:00:00 2001 From: Joshua Bronson Date: Mon, 30 Nov 2020 19:02:36 +0000 Subject: [PATCH 3/4] Address new review comments + rebase ...on top of latest master. Also drop now-removed `loop` kwarg from asyncio.sleep call. Ref: https://bugs.python.org/issue42392 --- Lib/operator.py | 14 ++++++++------ Lib/test/test_asyncgen.py | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Lib/operator.py b/Lib/operator.py index 9a07c50af99dab..f18cc8630963b1 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -21,7 +21,6 @@ ] from builtins import abs as _abs -from collections.abc import AsyncIterable, AsyncIterator # Comparison Operations *******************************************************# @@ -418,12 +417,14 @@ def aiter(obj, sentinel=_NOT_PROVIDED): Like the iter() builtin but for async iterables and callables. """ + from collections.abc import AsyncIterable, AsyncIterator if sentinel is _NOT_PROVIDED: if not isinstance(obj, AsyncIterable): raise TypeError(f'aiter expected an AsyncIterable, got {type(obj)}') - if isinstance(obj, AsyncIterator): - return obj - return (i async for i in obj) + ait = type(obj).__aiter__(obj) + if not isinstance(ait, AsyncIterator): + raise TypeError(f'obj.__aiter__() returned non-AsyncIterator: {type(ait)}') + return ait if not callable(obj): raise TypeError(f'aiter expected an async callable, got {type(obj)}') @@ -445,11 +446,12 @@ async def anext(async_iterator, default=_NOT_PROVIDED): If default is given and the iterator is exhausted, it is returned instead of raising StopAsyncIteration. """ + from collections.abc import AsyncIterator if not isinstance(async_iterator, AsyncIterator): raise TypeError(f'anext expected an AsyncIterator, got {type(async_iterator)}') - anxt = async_iterator.__anext__ + anxt = type(async_iterator).__anext__ try: - return await anxt() + return await anxt(async_iterator) except StopAsyncIteration: if default is _NOT_PROVIDED: raise diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 41d243094e6a41..77da57b39117d5 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -404,7 +404,7 @@ def test_async_gen_operator_aiter_class(self): class Gen: async def __aiter__(self): yield 1 - await asyncio.sleep(0.01, loop=loop) + await asyncio.sleep(0.01) yield 2 g = Gen() async def consume(): From 2f8df9945e698d8cb3d315671ef991209af91f8f Mon Sep 17 00:00:00 2001 From: Joshua Bronson Date: Fri, 4 Dec 2020 16:21:04 +0000 Subject: [PATCH 4/4] improve tests --- Lib/test/test_asyncgen.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 77da57b39117d5..5c2dad836ccb19 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -400,6 +400,7 @@ async def consume(): self.assertEqual(res, [1, 2]) def test_async_gen_operator_aiter_class(self): + results = [] loop = self.loop class Gen: async def __aiter__(self): @@ -408,9 +409,14 @@ async def __aiter__(self): yield 2 g = Gen() async def consume(): - return [i async for i in operator.aiter(g)] - res = self.loop.run_until_complete(consume()) - self.assertEqual(res, [1, 2]) + ait = operator.aiter(g) + while True: + try: + results.append(await operator.anext(ait)) + except StopAsyncIteration: + break + self.loop.run_until_complete(consume()) + self.assertEqual(results, [1, 2]) def test_async_gen_operator_aiter_2_arg(self): async def gen():