From ba711122b06b29929dcb6d3dc732b67ee14e434b Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Sun, 17 Apr 2016 19:03:53 +0100 Subject: [PATCH 01/66] Create Docker image and Docker Compose configuration --- Dockerfile | 26 ++++++++++++++++++++++++++ README.rst | 8 ++++++++ docker-compose.yml | 11 +++++++++++ docker-entrypoint.sh | 3 +++ 4 files changed, 48 insertions(+) create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100755 docker-entrypoint.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..6e897308 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM ubuntu:14.04 + +# From python:3.5 docker image, set locale +ENV LANG C.UTF-8 + +VOLUME /app +WORKDIR /app + +# Update base OS +RUN apt-get -y update && apt-get -y upgrade +# Install Python 3(.4) +RUN apt-get -y install python3 python3-dev python-virtualenv +# Install dependencies for Python libs +RUN apt-get -y install libxml2-dev libxslt1-dev zlib1g-dev + +# Copy needed files to build docker image +ADD requirements.txt docker-entrypoint.sh ./ + +# Create virtualenv +RUN virtualenv -p python3 /venv +# Populate virtualenv +RUN ./docker-entrypoint.sh pip install --upgrade pip +RUN ./docker-entrypoint.sh pip install -r requirements.txt + +ENTRYPOINT ["./docker-entrypoint.sh"] +CMD ["./run_csbot.py", "csbot.cfg"] diff --git a/README.rst b/README.rst index 7d22b1fe..9193c1c0 100644 --- a/README.rst +++ b/README.rst @@ -22,6 +22,13 @@ If you want to develop features for the bot, create a uniquely named plugin (see ``csbot/plugins/`` for examples), try it out, preferably write some unit tests (see ``csbot/test/plugins/``) and submit a pull request. +Deployment +---------- +Create ``csbot.cfg``, and then use `Docker Compose`_ to build and launch the +Docker containers (a MongoDB instance and the bot):: + + $ docker-compose up + Documentation ------------- The code is documented to varying degrees, and Sphinx-based documentation is @@ -57,3 +64,4 @@ We're also using Travis-CI for continuous integration and continuous deployment. .. _HackSoc: http://hacksoc.org/ .. _asyncio: https://docs.python.org/3/library/asyncio.html .. _lxml: http://lxml.de/ +.. _Docker Compose: https://docs.docker.com/compose/ diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..2fb36608 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,11 @@ +bot: + build: . + links: + - mongodb + volumes: + - .:/app + environment: + MONGODB_URI: mongodb://mongodb:27017/csbot + +mongodb: + image: mongo:3.2 diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 00000000..7e739d69 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/bash +. /venv/bin/activate +exec $@ From 3f00ba9f49c1846256392cba123d07d78113f226 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Thu, 31 Jan 2019 23:15:45 +0000 Subject: [PATCH 02/66] Update all the docker bits --- Dockerfile | 2 +- README.rst | 8 ++++++++ docker-compose.yml | 27 +++++++++++++++++---------- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6e897308..81f59578 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:14.04 +FROM ubuntu:18.04 # From python:3.5 docker image, set locale ENV LANG C.UTF-8 diff --git a/README.rst b/README.rst index 9193c1c0..9112b99d 100644 --- a/README.rst +++ b/README.rst @@ -29,6 +29,14 @@ Docker containers (a MongoDB instance and the bot):: $ docker-compose up +Backup MongoDB once services are running:: + + $ docker-compose exec -T mongodb mongodump --archive --gzip > foo.mongodump.gz + +Restore MongoDB:: + + $ docker-compose exec -T mongodb mongorestore --archive --gzip --drop < foo.mongodump.gz + Documentation ------------- The code is documented to varying degrees, and Sphinx-based documentation is diff --git a/docker-compose.yml b/docker-compose.yml index 2fb36608..231fc072 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,18 @@ -bot: - build: . - links: - - mongodb - volumes: - - .:/app - environment: - MONGODB_URI: mongodb://mongodb:27017/csbot +version: "3" -mongodb: - image: mongo:3.2 +services: + bot: + build: . + links: + - mongodb + volumes: + - .:/app + env_file: + - ./deploy.env + environment: + MONGODB_URI: mongodb://mongodb:27017/csbot + + mongodb: + image: mongo:4.1 + volumes: + - ./mongodb-data:/data/db From df9130542b77aa2c3c341a653a28e4486638c629 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Fri, 1 Feb 2019 08:26:44 +0000 Subject: [PATCH 03/66] Don't use unstable release of MongoDB... --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 231fc072..113d38e3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,6 +13,6 @@ services: MONGODB_URI: mongodb://mongodb:27017/csbot mongodb: - image: mongo:4.1 + image: mongo:4.0 volumes: - ./mongodb-data:/data/db From 33a355baa9ceba1cffbda1cb43731dec4ce9a13b Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Fri, 1 Feb 2019 08:27:09 +0000 Subject: [PATCH 04/66] Add oauth2client dependency that google-api-python-client dropped Fixes #144 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 8b8b3ce9..89781c2f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ pymongo>=3.6.0 requests>=2.9.1,<3.0.0 lxml>=2.3.5 google-api-python-client>=1.4.1,<2.0.0 +oauth2client>=3,<4 # google-api-python-client dropped dep in a minor release, generates ImportError warnings imgurpython>=1.1.6,<2.0.0 isodate>=0.5.1 aiohttp>=3.5.1,<4.0 From d0592c21c20e8c0cb68286f12e73246200d12d0a Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Fri, 1 Feb 2019 09:37:54 +0000 Subject: [PATCH 05/66] Fix mongodump instructions --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 9112b99d..d257d3e3 100644 --- a/README.rst +++ b/README.rst @@ -31,7 +31,7 @@ Docker containers (a MongoDB instance and the bot):: Backup MongoDB once services are running:: - $ docker-compose exec -T mongodb mongodump --archive --gzip > foo.mongodump.gz + $ docker-compose exec -T mongodb mongodump --archive --gzip --quiet > foo.mongodump.gz Restore MongoDB:: From 9676006b320a74f17f1bacca9f3e8ad7c560b544 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Fri, 1 Feb 2019 23:34:28 +0000 Subject: [PATCH 06/66] Updated docker-compose.yml for working deployment --- docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 113d38e3..1ba4756e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,9 @@ services: - ./deploy.env environment: MONGODB_URI: mongodb://mongodb:27017/csbot + command: ./run_csbot.py csbot.deploy.cfg --rollbar + ports: + - "127.0.0.1:8180:80" mongodb: image: mongo:4.0 From 5736423858db807f255f12e981882427adf77e5f Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Sat, 16 Feb 2019 20:32:53 +0000 Subject: [PATCH 07/66] Enable client PING --- csbot.deploy.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/csbot.deploy.cfg b/csbot.deploy.cfg index 118ba8b6..45b74d95 100644 --- a/csbot.deploy.cfg +++ b/csbot.deploy.cfg @@ -3,6 +3,7 @@ nickname = Mathison auth_method = sasl_plain channels = #cs-york #cs-york-dev #compsoc-uk #hacksoc plugins = logger linkinfo hoogle imgur csyork usertrack auth topic helix calc mongodb termdates whois xkcd youtube last webserver webhook github +client_ping = 60 [linkinfo] scan_limit = 2 From f81af0071b418d2ab9bf255823e68736cbd96b4f Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Wed, 22 May 2019 10:20:49 +0100 Subject: [PATCH 08/66] Add IRCClient.wait_for() to allow synchronising on IRC server responses --- csbot/irc.py | 27 +++++++++++++++++++ csbot/test/test_irc.py | 61 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) diff --git a/csbot/irc.py b/csbot/irc.py index 764ba6ce..021b5515 100644 --- a/csbot/irc.py +++ b/csbot/irc.py @@ -6,6 +6,7 @@ import codecs import base64 import types +from typing import Callable from ._rfc import NUMERIC_REPLIES @@ -259,6 +260,8 @@ def __init__(self, *, loop=None, **kwargs): self._client_ping = None self._client_ping_counter = 0 + self._waiters = set() + self.nick = self.__config['nick'] self.available_capabilities = set() self.enabled_capabilities = set() @@ -377,6 +380,13 @@ def line_received(self, line): def message_received(self, msg): """Callback for received parsed IRC message.""" + done = set() + for w in self._waiters: + if not w.future.cancelled() and w.predicate(msg): + w.future.set_result(msg) + if w.future.done(): + done.add(w) + self._waiters.difference_update(done) self._dispatch_method('irc_' + msg.command_name, msg) def send_line(self, data): @@ -413,6 +423,23 @@ async def _send_client_pings(self, interval): self._client_ping_counter += 1 self.send_line(f'PING {self._client_ping_counter}') + class Waiter: + predicate = None + future = None + + def __init__(self, predicate, future): + self.predicate = predicate + self.future = future + + def wait_for(self, predicate: Callable[[IRCMessage], bool]) -> asyncio.Future: + """Wait for a message that matches *predicate*. + + Returns a future that is resolved with the first message that matches *predicate*. + """ + waiter = self.Waiter(predicate, self.loop.create_future()) + self._waiters.add(waiter) + return waiter.future + # Specific commands for sending messages def enable_capability(self, name): diff --git a/csbot/test/test_irc.py b/csbot/test/test_irc.py index f77ac04a..a299b51a 100644 --- a/csbot/test/test_irc.py +++ b/csbot/test/test_irc.py @@ -307,6 +307,67 @@ def test_parse_failure(irc_client_helper): irc_client_helper.receive('') +@pytest.mark.asyncio +async def test_wait_for_success(irc_client_helper): + messages = [ + IRCMessage(None, 'PING', ['0'], 'PING', 'PING :0'), + IRCMessage(None, 'PING', ['1'], 'PING', 'PING :1'), + IRCMessage(None, 'PING', ['2'], 'PING', 'PING :2'), + ] + + mock_predicate = mock.Mock(return_value=False) + fut_mock = irc_client_helper.client.wait_for(mock_predicate) + + # Predicate is called, but future is not resolved + irc_client_helper.receive(messages[0].raw) + assert mock_predicate.mock_calls == [ + mock.call(messages[0]), + ] + assert not fut_mock.done() + + # Predicate is called, and future is resolved with matching message + mock_predicate.return_value = True + irc_client_helper.receive(messages[1].raw) + assert mock_predicate.mock_calls == [ + mock.call(messages[0]), + mock.call(messages[1]), + ] + assert fut_mock.done() + assert fut_mock.result() == messages[1] + + # Predicate is not called, because once resolved it was removed + irc_client_helper.receive(messages[2].raw) + assert mock_predicate.mock_calls == [ + mock.call(messages[0]), + mock.call(messages[1]), + ] + + +@pytest.mark.asyncio +async def test_wait_for_cancelled(irc_client_helper): + messages = [ + IRCMessage(None, 'PING', ['0'], 'PING', 'PING :0'), + IRCMessage(None, 'PING', ['1'], 'PING', 'PING :1'), + ] + + mock_predicate = mock.Mock(return_value=False) + fut_mock = irc_client_helper.client.wait_for(mock_predicate) + + # Predicate is called, but future is not resolved + irc_client_helper.receive(messages[0].raw) + assert mock_predicate.mock_calls == [ + mock.call(messages[0]), + ] + assert not fut_mock.done() + + # Predicate is not called, because future was cancelled + fut_mock.cancel() + irc_client_helper.receive(messages[1].raw) + assert mock_predicate.mock_calls == [ + mock.call(messages[0]), + ] + + # Test that calling various commands causes the appropriate messages to be sent to the server def test_set_nick(irc_client_helper): From dd476844de6d6a806e7d8de23a5c9f8a483b0965 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Wed, 22 May 2019 22:00:11 +0100 Subject: [PATCH 09/66] Allow IRCClient.wait_for() to pass through predicate exceptions --- csbot/irc.py | 10 ++++++++-- csbot/test/test_irc.py | 25 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/csbot/irc.py b/csbot/irc.py index 021b5515..9f7807a9 100644 --- a/csbot/irc.py +++ b/csbot/irc.py @@ -382,8 +382,14 @@ def message_received(self, msg): """Callback for received parsed IRC message.""" done = set() for w in self._waiters: - if not w.future.cancelled() and w.predicate(msg): - w.future.set_result(msg) + if not w.future.done(): + try: + matched = w.predicate(msg) + except Exception as e: + w.future.set_exception(e) + matched = False + if matched: + w.future.set_result(msg) if w.future.done(): done.add(w) self._waiters.difference_update(done) diff --git a/csbot/test/test_irc.py b/csbot/test/test_irc.py index a299b51a..93676998 100644 --- a/csbot/test/test_irc.py +++ b/csbot/test/test_irc.py @@ -368,6 +368,31 @@ async def test_wait_for_cancelled(irc_client_helper): ] +@pytest.mark.asyncio +async def test_wait_for_exception(irc_client_helper): + messages = [ + IRCMessage(None, 'PING', ['0'], 'PING', 'PING :0'), + IRCMessage(None, 'PING', ['1'], 'PING', 'PING :1'), + ] + + mock_predicate = mock.Mock(side_effect=Exception()) + fut_mock = irc_client_helper.client.wait_for(mock_predicate) + + # Predicate is called, but future has exception + irc_client_helper.receive(messages[0].raw) + assert mock_predicate.mock_calls == [ + mock.call(messages[0]), + ] + assert fut_mock.done() + assert fut_mock.exception() is not None + + # Predicate is not called, because future is already done + irc_client_helper.receive(messages[1].raw) + assert mock_predicate.mock_calls == [ + mock.call(messages[0]), + ] + + # Test that calling various commands causes the appropriate messages to be sent to the server def test_set_nick(irc_client_helper): From eab87e8c8db5564bd02c3db1697d59894345d801 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Thu, 23 May 2019 10:49:02 +0100 Subject: [PATCH 10/66] Make connection_made()/connection_lost() async, to allow for wait_for() --- csbot/core.py | 8 ++++---- csbot/irc.py | 8 ++++---- csbot/test/test_plugin_linkinfo.py | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/csbot/core.py b/csbot/core.py index e196c1c8..c7a67738 100644 --- a/csbot/core.py +++ b/csbot/core.py @@ -205,12 +205,12 @@ def emit(self, event): """ self.bot.post_event(event) - def connection_made(self): - super().connection_made() + async def connection_made(self): + await super().connection_made() self.emit_new('core.raw.connected') - def connection_lost(self, exc): - super().connection_lost(exc) + async def connection_lost(self, exc): + await super().connection_lost(exc) self.emit_new('core.raw.disconnected', {'reason': repr(exc)}) def send_line(self, line): diff --git a/csbot/irc.py b/csbot/irc.py index 9f7807a9..ef1d2fc4 100644 --- a/csbot/irc.py +++ b/csbot/irc.py @@ -273,9 +273,9 @@ async def run(self, run_once=False): await self.connect() self.connected.set() self.disconnected.clear() - self.connection_made() + await self.connection_made() await self.read_loop() - self.connection_lost(self.reader.exception()) + await self.connection_lost(self.reader.exception()) self.connected.clear() self.disconnected.set() if self._exiting: @@ -318,7 +318,7 @@ async def read_loop(self): break self.line_received(self.codec.decode(line[:-2])) - def connection_made(self): + async def connection_made(self): """Callback for successful connection. Register with the IRC server. @@ -362,7 +362,7 @@ def connection_made(self): self._start_client_pings() - def connection_lost(self, exc): + async def connection_lost(self, exc): """Handle a broken connection by attempting to reconnect. Won't reconnect if the broken connection was deliberate (i.e. diff --git a/csbot/test/test_plugin_linkinfo.py b/csbot/test/test_plugin_linkinfo.py index ac6c5cde..1d3a2c1a 100644 --- a/csbot/test/test_plugin_linkinfo.py +++ b/csbot/test/test_plugin_linkinfo.py @@ -133,8 +133,8 @@ @pytest.fixture -def irc_client(irc_client): - irc_client.connection_made() +async def irc_client(irc_client): + await irc_client.connection_made() return irc_client From ad932366d38095bb1475db9855442a155dca204d Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Thu, 23 May 2019 22:20:03 +0100 Subject: [PATCH 11/66] Rename wait_for to wait_for_message, let callback decide future result --- csbot/irc.py | 42 ++++++++++++++++++++++++------------------ csbot/test/test_irc.py | 14 +++++++------- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/csbot/irc.py b/csbot/irc.py index ef1d2fc4..f00ab9d0 100644 --- a/csbot/irc.py +++ b/csbot/irc.py @@ -6,7 +6,7 @@ import codecs import base64 import types -from typing import Callable +from typing import Callable, Tuple, Any, Iterable, Awaitable from ._rfc import NUMERIC_REPLIES @@ -260,7 +260,7 @@ def __init__(self, *, loop=None, **kwargs): self._client_ping = None self._client_ping_counter = 0 - self._waiters = set() + self._message_waiters = set() self.nick = self.__config['nick'] self.available_capabilities = set() @@ -380,19 +380,7 @@ def line_received(self, line): def message_received(self, msg): """Callback for received parsed IRC message.""" - done = set() - for w in self._waiters: - if not w.future.done(): - try: - matched = w.predicate(msg) - except Exception as e: - w.future.set_exception(e) - matched = False - if matched: - w.future.set_result(msg) - if w.future.done(): - done.add(w) - self._waiters.difference_update(done) + self.process_wait_for_message(msg) self._dispatch_method('irc_' + msg.command_name, msg) def send_line(self, data): @@ -437,15 +425,33 @@ def __init__(self, predicate, future): self.predicate = predicate self.future = future - def wait_for(self, predicate: Callable[[IRCMessage], bool]) -> asyncio.Future: + def wait_for_message(self, predicate: Callable[[IRCMessage], Tuple[bool, Any]]) -> asyncio.Future: """Wait for a message that matches *predicate*. - Returns a future that is resolved with the first message that matches *predicate*. + *predicate* should return a `(did_match, result)` tuple, where *did_match* is a boolean + indicating if the message is a match, and *result* is the value to return. + + Returns a future that is resolved with *result* on the first matching message. """ waiter = self.Waiter(predicate, self.loop.create_future()) - self._waiters.add(waiter) + self._message_waiters.add(waiter) return waiter.future + def process_wait_for_message(self, msg): + done = set() + for w in self._message_waiters: + if not w.future.done(): + matched, result = False, None + try: + matched, result = w.predicate(msg) + except Exception as e: + w.future.set_exception(e) + if matched: + w.future.set_result(result) + if w.future.done(): + done.add(w) + self._message_waiters.difference_update(done) + # Specific commands for sending messages def enable_capability(self, name): diff --git a/csbot/test/test_irc.py b/csbot/test/test_irc.py index 93676998..46c04e00 100644 --- a/csbot/test/test_irc.py +++ b/csbot/test/test_irc.py @@ -315,8 +315,8 @@ async def test_wait_for_success(irc_client_helper): IRCMessage(None, 'PING', ['2'], 'PING', 'PING :2'), ] - mock_predicate = mock.Mock(return_value=False) - fut_mock = irc_client_helper.client.wait_for(mock_predicate) + mock_predicate = mock.Mock(return_value=(False, None)) + fut_mock = irc_client_helper.client.wait_for_message(mock_predicate) # Predicate is called, but future is not resolved irc_client_helper.receive(messages[0].raw) @@ -326,14 +326,14 @@ async def test_wait_for_success(irc_client_helper): assert not fut_mock.done() # Predicate is called, and future is resolved with matching message - mock_predicate.return_value = True + mock_predicate.return_value = (True, 'foo') irc_client_helper.receive(messages[1].raw) assert mock_predicate.mock_calls == [ mock.call(messages[0]), mock.call(messages[1]), ] assert fut_mock.done() - assert fut_mock.result() == messages[1] + assert fut_mock.result() == 'foo' # Predicate is not called, because once resolved it was removed irc_client_helper.receive(messages[2].raw) @@ -350,8 +350,8 @@ async def test_wait_for_cancelled(irc_client_helper): IRCMessage(None, 'PING', ['1'], 'PING', 'PING :1'), ] - mock_predicate = mock.Mock(return_value=False) - fut_mock = irc_client_helper.client.wait_for(mock_predicate) + mock_predicate = mock.Mock(return_value=(False, None)) + fut_mock = irc_client_helper.client.wait_for_message(mock_predicate) # Predicate is called, but future is not resolved irc_client_helper.receive(messages[0].raw) @@ -376,7 +376,7 @@ async def test_wait_for_exception(irc_client_helper): ] mock_predicate = mock.Mock(side_effect=Exception()) - fut_mock = irc_client_helper.client.wait_for(mock_predicate) + fut_mock = irc_client_helper.client.wait_for_message(mock_predicate) # Predicate is called, but future has exception irc_client_helper.receive(messages[0].raw) From 83c7c3a4ef5291a7d90104a01a9f3306e751449d Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Fri, 24 May 2019 10:04:42 +0100 Subject: [PATCH 12/66] Make IRCv3 features optional, synchronise on capability requests --- csbot/core.py | 12 +++---- csbot/irc.py | 89 ++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 75 insertions(+), 26 deletions(-) diff --git a/csbot/core.py b/csbot/core.py index c7a67738..74da6bea 100644 --- a/csbot/core.py +++ b/csbot/core.py @@ -23,6 +23,7 @@ class Bot(SpecialPlugin, IRCClient): #: Default configuration values CONFIG_DEFAULTS = { + 'ircv3': False, 'nickname': 'csyorkbot', 'password': None, 'auth_method': 'pass', @@ -71,6 +72,7 @@ def __init__(self, config=None, loop=None): IRCClient.__init__( self, loop=loop, + ircv3=self.config_getboolean('ircv3'), nick=self.config_get('nickname'), username=self.config_get('username'), host=self.config_get('irc_host'), @@ -207,6 +209,8 @@ def emit(self, event): async def connection_made(self): await super().connection_made() + if self.config_getboolean('ircv3'): + await self.request_capabilities(enable={'account-notify', 'extended-join'}) self.emit_new('core.raw.connected') async def connection_lost(self, exc): @@ -328,14 +332,6 @@ def on_names(self, channel, names, raw_names): 'raw_names': raw_names, }) - # "IRC Client Capabilities" - - def on_capabilities_available(self, capabilities): - super().on_capabilities_available(capabilities) - for cap in ['account-notify', 'extended-join']: - if cap in capabilities and cap not in self.enabled_capabilities: - self.enable_capability(cap) - # Implement active account discovery via "formatted WHO" def identify(self, target): diff --git a/csbot/irc.py b/csbot/irc.py index f00ab9d0..09a5be3c 100644 --- a/csbot/irc.py +++ b/csbot/irc.py @@ -191,6 +191,10 @@ def decode(self, input, errors='strict'): return codecs.decode(input, 'cp1252', 'replace') +class IRCClientError(Exception): + pass + + class IRCClient: """Internet Relay Chat client protocol. @@ -229,6 +233,7 @@ class IRCClient: #: Generate a default configuration. Easier to call this and update the #: result than relying on ``dict.copy()``. DEFAULTS = staticmethod(lambda: dict( + ircv3=False, nick='csbot', username=None, host='irc.freenode.net', @@ -273,8 +278,10 @@ async def run(self, run_once=False): await self.connect() self.connected.set() self.disconnected.clear() + # Need to start read_loop() first so that connection_made() can await messages + read_loop_fut = self.loop.create_task(self.read_loop()) await self.connection_made() - await self.read_loop() + await read_loop_fut await self.connection_lost(self.reader.exception()) self.connected.clear() self.disconnected.set() @@ -331,34 +338,40 @@ async def connection_made(self): password = self.__config['password'] auth_method = self.__config['auth_method'] + if self.__config['ircv3']: + # Discover available capabilities + self.send(IRCMessage.create('CAP', ['LS'])) + await self.wait_for_message(lambda m: (m.command == 'CAP' and m.params[1] == 'LS', m)) + if auth_method == 'pass': if password: self.send(IRCMessage.create('PASS', [password])) self.set_nick(nick) self.send(user_msg) elif auth_method == 'sasl_plain': - # Just assume the server is going to understand our attempt at SASL - # authentication... - # TODO: proper stateful capability negotiation at this step - self.enable_capability('sasl') + sasl_enabled = await self.request_capabilities(enable={'sasl'}) self.set_nick(nick) self.send(user_msg) - self.send(IRCMessage.create('AUTHENTICATE', ['PLAIN'])) - # SASL PLAIN authentication message (https://tools.ietf.org/html/rfc4616) - # (assuming authzid = authcid = nick) - sasl_plain = '{}\0{}\0{}'.format(nick, nick, password) - # Well this is awkward... password string encoded to bytes as utf-8, - # base64-encoded to different bytes, converted back to string for - # use in the IRCMessage (which later encodes it as utf-8...) - sasl_plain_b64 = base64.b64encode(sasl_plain.encode('utf-8')).decode('ascii') - self.send(IRCMessage.create('AUTHENTICATE', [sasl_plain_b64])) - self.send(IRCMessage.create('CAP', ['END'])) + if sasl_enabled: + self.send(IRCMessage.create('AUTHENTICATE', ['PLAIN'])) + # SASL PLAIN authentication message (https://tools.ietf.org/html/rfc4616) + # (assuming authzid = authcid = nick) + sasl_plain = '{}\0{}\0{}'.format(nick, nick, password) + # Well this is awkward... password string encoded to bytes as utf-8, + # base64-encoded to different bytes, converted back to string for + # use in the IRCMessage (which later encodes it as utf-8...) + sasl_plain_b64 = base64.b64encode(sasl_plain.encode('utf-8')).decode('ascii') + self.send(IRCMessage.create('AUTHENTICATE', [sasl_plain_b64])) + else: + LOG.error('could not enable "sasl" capability, skipping authentication') else: raise ValueError('unknown auth_method: {}'.format(auth_method)) - # Discover available client capabilities, if any, which should get - # enabled in callbacks triggered by the CAP LS response - self.send_line('CAP LS') + if self.__config['ircv3']: + self.send(IRCMessage.create('CAP', ['END'])) + + # TODO: uncomment this? tests hang if uncommented... + # await self.wait_for_message(lambda m: (m.command_name == 'RPL_WELCOME', m)) self._start_client_pings() @@ -454,6 +467,46 @@ def process_wait_for_message(self, msg): # Specific commands for sending messages + def request_capabilities(self, *, enable: Iterable[str] = None, disable: Iterable[str] = None) -> Awaitable[bool]: + """Request a change to the enabled IRCv3 capabilities. + + *enable* and *disable* are sets of capability names, with *disable* taking precedence. + + Returns a future which resolves with True if the request is successful, or False otherwise. + """ + if not self.__config['ircv3']: + raise IRCClientError('configured with ircv3=False, cannot use capability negotiation') + + enable_set = set(enable or ()) + disable_set = set(disable or ()) + enable_set.difference_update(disable_set) + unknown = enable_set.union(disable_set).difference(self.available_capabilities) + if unknown: + LOG.warning('attempting to request unknown capabilities: %r', unknown) + + request = ' '.join(sorted(enable_set) + [f'-{c}' for c in sorted(disable_set)]) + if len(request) == 0: + LOG.warning('no capabilities requested, not sending CAP REQ') + fut = self.loop.create_future() + fut.set_result(True) + return fut + else: + message = IRCMessage.create('CAP', ['REQ', request]) + self.send(message) + return self._wait_for_capability_response(request) + + def _wait_for_capability_response(self, request): + def predicate(msg): + if msg.command == 'CAP': + _, subcommand, response = msg.params + response = response.strip() + if subcommand == 'ACK' and response == request: + return True, True + elif subcommand == 'NAK' and response == request: + return True, False + return False, None + return self.wait_for_message(predicate) + def enable_capability(self, name): """Enable client capability *name*. From 6ada38901e224168cf07ecc62845f6519afcc144 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Fri, 24 May 2019 10:19:25 +0100 Subject: [PATCH 13/66] Wait for SASL authentication response --- csbot/irc.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/csbot/irc.py b/csbot/irc.py index 09a5be3c..c9d70795 100644 --- a/csbot/irc.py +++ b/csbot/irc.py @@ -362,6 +362,9 @@ async def connection_made(self): # use in the IRCMessage (which later encodes it as utf-8...) sasl_plain_b64 = base64.b64encode(sasl_plain.encode('utf-8')).decode('ascii') self.send(IRCMessage.create('AUTHENTICATE', [sasl_plain_b64])) + sasl_success = await self.wait_for_message(lambda m: (m.command in ('903', '904'), m.command == '903')) + if not sasl_success: + LOG.error('SASL authentication failed') else: LOG.error('could not enable "sasl" capability, skipping authentication') else: From 65d01a42d961c792f6910bbb015958c5aa5a32a7 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Mon, 27 May 2019 17:21:09 +0100 Subject: [PATCH 14/66] Remove less useful capability negotiation methods --- csbot/irc.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/csbot/irc.py b/csbot/irc.py index c9d70795..45d5d335 100644 --- a/csbot/irc.py +++ b/csbot/irc.py @@ -510,26 +510,6 @@ def predicate(msg): return False, None return self.wait_for_message(predicate) - def enable_capability(self, name): - """Enable client capability *name*. - - Should wait for :meth:`on_capability_enabled` before assuming it is - enabled. - """ - if name not in self.available_capabilities: - LOG.warning('Enabling client capability "{}" not in response to CAP LS'.format(name)) - self.send_line('CAP REQ :{}'.format(name)) - - def disable_capability(self, name): - """Disable client capability *name*. - - Should wait for :meth:`on_capability_disabled` befor assuming it is - disabled. - """ - if name not in self.available_capabilities: - LOG.warning('Disabling client capability "{}" not in response to CAP LS'.format(name)) - self.send_line('CAP REQ :-{}'.format(name)) - def set_nick(self, nick): """Ask the server to set our nick.""" self.send_line('NICK {}'.format(nick)) From bc9708283691038e2926805ebd8238968444f96c Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Sun, 26 May 2019 15:04:31 +0100 Subject: [PATCH 15/66] WIP: add tests for event ordering --- csbot/test/conftest.py | 3 +- csbot/test/test_bot.py | 66 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 csbot/test/test_bot.py diff --git a/csbot/test/conftest.py b/csbot/test/conftest.py index 7437d728..048c4dc1 100644 --- a/csbot/test/conftest.py +++ b/csbot/test/conftest.py @@ -105,8 +105,7 @@ def receive(self, lines): """Shortcut to push a series of lines to the client.""" if isinstance(lines, str): lines = [lines] - for l in lines: - self.client.line_received(l) + return [self.client.line_received(l) for l in lines] def assert_sent(self, lines): """Check that a list of (unicode) strings have been sent. diff --git a/csbot/test/test_bot.py b/csbot/test/test_bot.py new file mode 100644 index 00000000..3f5b7c16 --- /dev/null +++ b/csbot/test/test_bot.py @@ -0,0 +1,66 @@ +import unittest.mock as mock +import asyncio + +import pytest + +from csbot import core +from csbot.plugin import Plugin + + +class TestHookOrdering: + class Bot(core.Bot): + class MockPlugin(Plugin): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.handler_mock = mock.Mock(spec=callable) + + @Plugin.hook('core.message.privmsg') + async def privmsg(self, event): + await asyncio.sleep(0.5) + self.handler_mock('privmsg', event['message']) + + @Plugin.hook('core.user.quit') + def quit(self, event): + self.handler_mock('quit', event['user']) + + available_plugins = core.Bot.available_plugins.copy() + available_plugins.update( + mockplugin=MockPlugin, + ) + + CONFIG = f"""\ + [@bot] + plugins = mockplugin + """ + pytestmark = pytest.mark.bot(cls=Bot, config=CONFIG) + + @pytest.mark.asyncio + @pytest.mark.parametrize('n', list(range(1, 10))) + async def test_burst_in_order(self, bot_helper, n): + """Check that a plugin always gets messages in receive order.""" + plugin = bot_helper['mockplugin'] + users = [f':nick{i}!user{i}@host{i}' for i in range(n)] + messages = [f':{user} QUIT :*.net *.split' for user in users] + await asyncio.wait(bot_helper.receive(messages)) + assert plugin.handler_mock.mock_calls == [mock.call('quit', user) for user in users] + + @pytest.mark.asyncio + async def test_non_blocking(self, bot_helper): + plugin = bot_helper['mockplugin'] + messages = [ + ':nick0!user@host QUIT :bye', + ':nick1!user@host QUIT :bye', + ':foo!user@host PRIVMSG #channel :hello', + ':nick2!user@host QUIT :bye', + ':nick3!user@host QUIT :bye', + ':nick4!user@host QUIT :bye', + ] + await asyncio.wait(bot_helper.receive(messages)) + assert plugin.handler_mock.mock_calls == [ + mock.call('quit', 'nick0!user@host'), + mock.call('quit', 'nick1!user@host'), + mock.call('quit', 'nick2!user@host'), + mock.call('quit', 'nick3!user@host'), + mock.call('quit', 'nick4!user@host'), + mock.call('privmsg', 'hello'), + ] From a4522f6c47cb91e08b65f65cffb15284bd7ac417 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Tue, 28 May 2019 16:11:33 +0100 Subject: [PATCH 16/66] Implement and test HybridEventRunner --- csbot/events.py | 78 +++++++++ csbot/test/test_events.py | 353 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 431 insertions(+) diff --git a/csbot/events.py b/csbot/events.py index c7692e0c..19476857 100644 --- a/csbot/events.py +++ b/csbot/events.py @@ -159,6 +159,84 @@ def _run(self): not_done.add(new_pending) +class HybridEventRunner: + def __init__(self, handle_event, loop=None): + self.handle_event = handle_event + self.loop = loop + + self.events = deque() + self.new_events = asyncio.Event(loop=self.loop) + self.futures = set() + self.future = None + + def __enter__(self): + LOG.debug('entering event runner') + + def __exit__(self, exc_type, exc_value, traceback): + LOG.debug('exiting event runner') + self.future = None + + def post_event(self, event): + self.events.append(event) + LOG.debug('added event %s, pending=%s', event, len(self.events)) + self.new_events.set() + if not self.future: + self.future = self.loop.create_task(self._run()) + return self.future + + def _run_events(self): + new_futures = set() + while len(self.events) > 0: + LOG.debug('processing events (%s remaining)', len(self.events)) + # Get next event + event = self.events.popleft() + # Handle the event + results = self.handle_event(event) + # Schedule the awaitables + for r in results: + if r is None: + continue + try: + f = asyncio.ensure_future(r, loop=self.loop) + except TypeError: + LOG.exception('non-awaitable result %r handling event %r', r, event) + continue + new_futures.add(f) + self.new_events.clear() + if len(new_futures) > 0: + LOG.debug('got %s new futures', len(new_futures)) + return new_futures + + async def _run(self): + # Use self as context manager so an escaping exception doesn't break + # the event runner instance permanently (i.e. we clean up the future) + with self: + # Run until no more events or lingering futures + while len(self.events) + len(self.futures) > 0: + # Synchronously run event handler and collect new futures + new_futures = self._run_events() + self.futures |= new_futures + # Don't bother waiting if no futures to wait on + if len(self.futures) == 0: + continue + + # Run until one or more futures complete (or new events are added) + new_events = self.loop.create_task(self.new_events.wait()) + LOG.debug('waiting on %s futures', len(self.futures)) + done, pending = await asyncio.wait(self.futures | {new_events}, + loop=self.loop, + return_when=asyncio.FIRST_COMPLETED) + # Remove done futures from the set of futures being waited on + done_futures = done - {new_events} + LOG.debug('%s of %s futures done', len(done_futures), len(self.futures)) + self.futures -= done_futures + if new_events.done(): + LOG.debug('new events to process') + else: + # If no new events, cancel the waiter, because we'll create a new one next iteration + new_events.cancel() + + class Event(dict): """IRC event information. diff --git a/csbot/test/test_events.py b/csbot/test/test_events.py index 44a90609..67f486ae 100644 --- a/csbot/test/test_events.py +++ b/csbot/test/test_events.py @@ -2,6 +2,8 @@ from unittest import mock import datetime import collections.abc +from collections import defaultdict +from functools import partial import asyncio import pytest @@ -185,6 +187,357 @@ def f4(): #self.assertEqual(set(self.handled_events), {f1, f2, f3, f4}) +@pytest.mark.asyncio +class TestHybridEventRunner: + class EventHandler: + def __init__(self): + self.handlers = defaultdict(list) + + def add(self, e, f=None): + if f is None: + return partial(self.add, e) + else: + self.handlers[e].append(f) + + def __call__(self, e): + return [f(e) for f in self.handlers[e]] + + @pytest.fixture + def event_runner(self, event_loop): + handler = self.EventHandler() + obj = mock.Mock() + obj.add_handler = handler.add + obj.handle_event = mock.Mock(wraps=handler) + obj.runner = csbot.events.HybridEventRunner(obj.handle_event, event_loop) + obj.exception_handler = mock.Mock(wraps=event_loop.get_exception_handler()) + event_loop.set_exception_handler(obj.exception_handler) + return obj + + async def test_values(self, event_runner): + """Check that basic values are passed through the event queue unmolested.""" + # Test that things actually get through + await event_runner.runner.post_event('foo') + assert event_runner.handle_event.call_args_list == [mock.call('foo')] + # The event runner doesn't care what it's passing through + for x in ['bar', 1.3, None, object]: + await event_runner.runner.post_event(x) + print(event_runner.handle_event.call_args) + assert event_runner.handle_event.call_args == mock.call(x) + + async def test_event_chain_synchronous(self, event_runner): + """Check that an entire event chain runs (synchronously). + + All handlers for an event should be run before the next event, and any events that occur + during an event handler should also be processed before the initial `post_event()` future + has a result. + """ + complete = [] + + @event_runner.add_handler('a') + def a(_): + event_runner.runner.post_event('b') + complete.append('a') + + @event_runner.add_handler('b') + def b1(_): + event_runner.runner.post_event('c') + complete.append('b1') + + @event_runner.add_handler('b') + def b2(_): + event_runner.runner.post_event('d') + complete.append('b2') + + @event_runner.add_handler('b') + def b3(_): + event_runner.runner.post_event('e') + complete.append('b3') + + @event_runner.add_handler('c') + def c(_): + event_runner.runner.post_event('f') + complete.append('c') + + @event_runner.add_handler('d') + def d(_): + complete.append('d') + + @event_runner.add_handler('e') + def e(_): + complete.append('e') + + await event_runner.runner.post_event('a') + assert event_runner.handle_event.mock_calls == [ + # Initial event + mock.call('a'), + # Event resulting from handler for 'a' + mock.call('b'), + # Ensure all handlers for 'b' finished ... + mock.call('c'), + mock.call('d'), + mock.call('e'), + # ... before first handler for 'c' + mock.call('f'), + ] + assert complete == ['a', 'b1', 'b2', 'b3', 'c', 'd', 'e'] + + async def test_event_chain_asynchronous(self, event_loop, event_runner): + """Check that an entire event chain runs (asynchronously). + + Any events that occur during an event handler should be processed before the initial + `post_event()` future has a result. + """ + events = [asyncio.Event(loop=event_loop) for _ in range(2)] + complete = [] + + @event_runner.add_handler('a') + async def a1(_): + complete.append('a1') + + @event_runner.add_handler('a') + async def a2(_): + await events[0].wait() + event_runner.runner.post_event('b') + complete.append('a2') + + @event_runner.add_handler('b') + async def b1(_): + event_runner.runner.post_event('c') + complete.append('b1') + + @event_runner.add_handler('b') + async def b2(_): + event_runner.runner.post_event('d') + complete.append('b2') + + @event_runner.add_handler('b') + async def b3(_): + await events[1].wait() + event_runner.runner.post_event('e') + complete.append('b3') + + @event_runner.add_handler('c') + async def c(_): + event_runner.runner.post_event('f') + complete.append('c') + + @event_runner.add_handler('d') + async def d(_): + complete.append('d') + + @event_runner.add_handler('e') + async def e(_): + complete.append('e') + + # Post the first event and allow some tasks to run: + # - should have a post_event('a') call + # - a1 should complete, a2 is blocked on events[0] + future = event_runner.runner.post_event('a') + await asyncio.wait({future}, loop=event_loop, timeout=0.1) + assert not future.done() + assert event_runner.handle_event.mock_calls == [ + mock.call('a'), + ] + assert complete == ['a1'] + + # Unblock a2 and allow some tasks to run: + # - a2 should complete + # - post_event('b') should be called (by a2) + # - b1 and b2 should complete, b3 is blocked on events[1] + # - post_event('c') and post_event('d') should be called (by b1 and b2) + # - c should complete + # - post_event('f') should be called (by c) + # - d should complete + events[0].set() + await asyncio.wait({future}, loop=event_loop, timeout=0.1) + assert not future.done() + assert event_runner.handle_event.mock_calls == [ + mock.call('a'), + mock.call('b'), + mock.call('c'), + mock.call('d'), + mock.call('f'), + ] + assert complete == ['a1', 'a2', 'b1', 'b2', 'c', 'd'] + + # Unblock b3 and allow some tasks to run: + # - b3 should complete + # - post_event('e') should be called (by b3) + # - e should complete + # - future should complete, because no events or tasks remain pending + events[1].set() + await asyncio.wait({future}, loop=event_loop, timeout=0.1) + assert future.done() + assert event_runner.handle_event.mock_calls == [ + mock.call('a'), + mock.call('b'), + mock.call('c'), + mock.call('d'), + mock.call('f'), + mock.call('e'), + ] + assert complete == ['a1', 'a2', 'b1', 'b2', 'c', 'd', 'b3', 'e'] + + async def test_event_chain_hybrid(self, event_loop, event_runner): + """Check that an entire event chain runs (mix of sync and async handlers). + + Synchronous handlers complete before asynchronous handlers. Synchronous handlers for an + event all run before synchronous handlers for the next event, but asynchronous handers can + run out-of-order. + """ + events = [asyncio.Event(loop=event_loop) for _ in range(2)] + complete = [] + + @event_runner.add_handler('a') + def a1(_): + complete.append('a1') + + @event_runner.add_handler('a') + async def a2(_): + await events[0].wait() + event_runner.runner.post_event('b') + complete.append('a2') + + @event_runner.add_handler('b') + async def b1(_): + await events[1].wait() + event_runner.runner.post_event('c') + complete.append('b1') + + @event_runner.add_handler('b') + def b2(_): + event_runner.runner.post_event('d') + complete.append('b2') + + @event_runner.add_handler('c') + def c1(_): + complete.append('c1') + + @event_runner.add_handler('c') + async def c2(_): + complete.append('c2') + + @event_runner.add_handler('d') + async def d1(_): + complete.append('d1') + + @event_runner.add_handler('d') + def d2(_): + complete.append('d2') + + # Post the first event and allow some tasks to run: + # - post_event('a') should be called (initial) + # - a1 should complete, a2 is blocked on events[0] + future = event_runner.runner.post_event('a') + await asyncio.wait({future}, loop=event_loop, timeout=0.1) + assert not future.done() + assert event_runner.handle_event.mock_calls == [ + mock.call('a'), + ] + assert complete == ['a1'] + + # Unblock a2 and allow some tasks to run: + # - a2 should complete + # - post_event('b') should be called (by a2) + # - b2 should complete, b1 is blocked on events[1] + # - post_event('d') should be called + # - d2 should complete (synchronous phase) + # - d1 should complete (asynchronous phase) + events[0].set() + await asyncio.wait({future}, loop=event_loop, timeout=0.1) + assert not future.done() + assert event_runner.handle_event.mock_calls == [ + mock.call('a'), + mock.call('b'), + mock.call('d'), + ] + assert complete == ['a1', 'a2', 'b2', 'd2', 'd1'] + + # Unblock b1 and allow some tasks to run: + # - b1 should complete + # - post_event('c') should be called (by b1) + # - c1 should complete (synchronous phase) + # - c2 should complete (asynchronous phase) + # - future should complete, because no events or tasks remain pending + events[1].set() + await asyncio.wait({future}, loop=event_loop, timeout=0.1) + assert future.done() + assert event_runner.handle_event.mock_calls == [ + mock.call('a'), + mock.call('b'), + mock.call('d'), + mock.call('c'), + ] + assert complete == ['a1', 'a2', 'b2', 'd2', 'd1', 'b1', 'c1', 'c2'] + + async def test_overlapping_root_events(self, event_loop, event_runner): + """Check that overlapping events get the same future.""" + events = [asyncio.Event(loop=event_loop) for _ in range(1)] + complete = [] + + @event_runner.add_handler('a') + async def a(_): + await events[0].wait() + complete.append('a') + + @event_runner.add_handler('b') + async def b(_): + complete.append('b') + + # Post the first event and allow tasks to run: + # - a is blocked on events[0] + f1 = event_runner.runner.post_event('a') + await asyncio.wait({f1}, loop=event_loop, timeout=0.1) + assert not f1.done() + assert complete == [] + + # Post the second event and allow tasks to run: + # - b completes + # - a is still blocked on events[0] + # - f1 and f2 are not done, because they're for the same run loop, and a is still blocked + f2 = event_runner.runner.post_event('b') + await asyncio.wait({f2}, loop=event_loop, timeout=0.1) + assert not f2.done() + assert not f1.done() + assert complete == ['b'] + + # Unblock a and allow tasks to run: + # - a completes + # - f1 and f2 are both done, because the run loop has finished + events[0].set() + await asyncio.wait([f1, f2], loop=event_loop, timeout=0.1) + assert f1.done() + assert f2.done() + assert complete == ['b', 'a'] + + # (Maybe remove this - not essential that they're the same future, only that they complete together) + assert f2 is f1 + + async def test_non_overlapping_root_events(self, event_loop, event_runner): + """Check that non-overlapping events get new futures.""" + complete = [] + + @event_runner.add_handler('a') + async def a(_): + complete.append('a') + + @event_runner.add_handler('b') + async def b(_): + complete.append('b') + + f1 = event_runner.runner.post_event('a') + await asyncio.wait({f1}, loop=event_loop, timeout=0.1) + assert f1.done() + assert complete == ['a'] + + f2 = event_runner.runner.post_event('b') + assert not f2.done() + assert f2 is not f1 + await asyncio.wait({f2}, loop=event_loop, timeout=0.1) + assert f2.done() + assert complete == ['a', 'b'] + + class TestEvent(unittest.TestCase): class DummyBot(object): pass From 5b6731813f822c9da53919227249d75574919cc0 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Tue, 28 May 2019 17:22:50 +0100 Subject: [PATCH 17/66] Move event handler calling into HybridEventRunner for consistent exception handling --- csbot/events.py | 35 +++++++++++++++----- csbot/test/test_events.py | 70 +++++++++++++++++++++++++++++++-------- 2 files changed, 83 insertions(+), 22 deletions(-) diff --git a/csbot/events.py b/csbot/events.py index 19476857..61a492e9 100644 --- a/csbot/events.py +++ b/csbot/events.py @@ -160,8 +160,8 @@ def _run(self): class HybridEventRunner: - def __init__(self, handle_event, loop=None): - self.handle_event = handle_event + def __init__(self, get_handlers, loop=None): + self.get_handlers = get_handlers self.loop = loop self.events = deque() @@ -191,17 +191,22 @@ def _run_events(self): # Get next event event = self.events.popleft() # Handle the event - results = self.handle_event(event) - # Schedule the awaitables - for r in results: - if r is None: + for handler in self.get_handlers(event): + # Attempt to run the handler, but don't break everything if the handler fails + try: + result = handler(event) + except Exception as e: + self._handle_exception(exception=e) + continue + # If the handler returned an awaitable (e.g. coroutine object), try to schedule it + if result is None: continue try: - f = asyncio.ensure_future(r, loop=self.loop) + future = asyncio.ensure_future(result, loop=self.loop) except TypeError: - LOG.exception('non-awaitable result %r handling event %r', r, event) + LOG.exception('non-awaitable result %r handling event %r', result, event) continue - new_futures.add(f) + new_futures.add(future) self.new_events.clear() if len(new_futures) > 0: LOG.debug('got %s new futures', len(new_futures)) @@ -230,12 +235,24 @@ async def _run(self): done_futures = done - {new_events} LOG.debug('%s of %s futures done', len(done_futures), len(self.futures)) self.futures -= done_futures + for f in done_futures: + if f.exception() is not None: + self._handle_exception(future=f) if new_events.done(): LOG.debug('new events to process') else: # If no new events, cancel the waiter, because we'll create a new one next iteration new_events.cancel() + def _handle_exception(self, *, message='Unhandled exception in event handler', exception=None, future=None): + if exception is None and future is not None: + exception = future.exception() + self.loop.call_exception_handler({ + 'message': message, + 'exception': exception, + 'future': future, + }) + class Event(dict): """IRC event information. diff --git a/csbot/test/test_events.py b/csbot/test/test_events.py index 67f486ae..eeb38749 100644 --- a/csbot/test/test_events.py +++ b/csbot/test/test_events.py @@ -200,15 +200,15 @@ def add(self, e, f=None): self.handlers[e].append(f) def __call__(self, e): - return [f(e) for f in self.handlers[e]] + return self.handlers[e] @pytest.fixture def event_runner(self, event_loop): handler = self.EventHandler() obj = mock.Mock() obj.add_handler = handler.add - obj.handle_event = mock.Mock(wraps=handler) - obj.runner = csbot.events.HybridEventRunner(obj.handle_event, event_loop) + obj.get_handlers = mock.Mock(wraps=handler) + obj.runner = csbot.events.HybridEventRunner(obj.get_handlers, event_loop) obj.exception_handler = mock.Mock(wraps=event_loop.get_exception_handler()) event_loop.set_exception_handler(obj.exception_handler) return obj @@ -217,12 +217,12 @@ async def test_values(self, event_runner): """Check that basic values are passed through the event queue unmolested.""" # Test that things actually get through await event_runner.runner.post_event('foo') - assert event_runner.handle_event.call_args_list == [mock.call('foo')] + assert event_runner.get_handlers.call_args_list == [mock.call('foo')] # The event runner doesn't care what it's passing through for x in ['bar', 1.3, None, object]: await event_runner.runner.post_event(x) - print(event_runner.handle_event.call_args) - assert event_runner.handle_event.call_args == mock.call(x) + print(event_runner.get_handlers.call_args) + assert event_runner.get_handlers.call_args == mock.call(x) async def test_event_chain_synchronous(self, event_runner): """Check that an entire event chain runs (synchronously). @@ -267,7 +267,7 @@ def e(_): complete.append('e') await event_runner.runner.post_event('a') - assert event_runner.handle_event.mock_calls == [ + assert event_runner.get_handlers.mock_calls == [ # Initial event mock.call('a'), # Event resulting from handler for 'a' @@ -335,7 +335,7 @@ async def e(_): future = event_runner.runner.post_event('a') await asyncio.wait({future}, loop=event_loop, timeout=0.1) assert not future.done() - assert event_runner.handle_event.mock_calls == [ + assert event_runner.get_handlers.mock_calls == [ mock.call('a'), ] assert complete == ['a1'] @@ -351,7 +351,7 @@ async def e(_): events[0].set() await asyncio.wait({future}, loop=event_loop, timeout=0.1) assert not future.done() - assert event_runner.handle_event.mock_calls == [ + assert event_runner.get_handlers.mock_calls == [ mock.call('a'), mock.call('b'), mock.call('c'), @@ -368,7 +368,7 @@ async def e(_): events[1].set() await asyncio.wait({future}, loop=event_loop, timeout=0.1) assert future.done() - assert event_runner.handle_event.mock_calls == [ + assert event_runner.get_handlers.mock_calls == [ mock.call('a'), mock.call('b'), mock.call('c'), @@ -431,7 +431,7 @@ def d2(_): future = event_runner.runner.post_event('a') await asyncio.wait({future}, loop=event_loop, timeout=0.1) assert not future.done() - assert event_runner.handle_event.mock_calls == [ + assert event_runner.get_handlers.mock_calls == [ mock.call('a'), ] assert complete == ['a1'] @@ -446,7 +446,7 @@ def d2(_): events[0].set() await asyncio.wait({future}, loop=event_loop, timeout=0.1) assert not future.done() - assert event_runner.handle_event.mock_calls == [ + assert event_runner.get_handlers.mock_calls == [ mock.call('a'), mock.call('b'), mock.call('d'), @@ -462,7 +462,7 @@ def d2(_): events[1].set() await asyncio.wait({future}, loop=event_loop, timeout=0.1) assert future.done() - assert event_runner.handle_event.mock_calls == [ + assert event_runner.get_handlers.mock_calls == [ mock.call('a'), mock.call('b'), mock.call('d'), @@ -537,6 +537,50 @@ async def b(_): assert f2.done() assert complete == ['a', 'b'] + @pytest.mark.asyncio(allow_unhandled_exception=True) + async def test_exception_recovery(self, event_loop, event_runner): + complete = [] + + @event_runner.add_handler('a') + def a1(_): + raise Exception('a1') + complete.append('a1') + + @event_runner.add_handler('a') + def a2(_): + complete.append('a2') + + @event_runner.add_handler('a') + async def a3(_): + raise Exception('a3') + complete.append('a3') + + @event_runner.add_handler('a') + async def a4(_): + event_runner.runner.post_event('b') + complete.append('a4') + + @event_runner.add_handler('b') + def b1(_): + raise Exception('b1') + complete.append('b1') + + @event_runner.add_handler('b') + async def b2(_): + complete.append('b2') + + assert event_runner.exception_handler.call_count == 0 + future = event_runner.runner.post_event('a') + await asyncio.wait({future}, loop=event_loop, timeout=0.1) + assert future.done() + assert future.exception() is None + assert event_runner.runner.get_handlers.mock_calls == [ + mock.call('a'), + mock.call('b'), + ] + assert complete == ['a2', 'a4', 'b2'] + assert event_runner.exception_handler.call_count == 3 + class TestEvent(unittest.TestCase): class DummyBot(object): From f52d94a5265288af88fd103cfabdb6ac1703bb83 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Tue, 28 May 2019 20:11:07 +0100 Subject: [PATCH 18/66] Use HybridEventRunner in Bot --- csbot/core.py | 8 +++----- csbot/plugin.py | 16 ++++------------ 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/csbot/core.py b/csbot/core.py index 74da6bea..367449d9 100644 --- a/csbot/core.py +++ b/csbot/core.py @@ -1,6 +1,5 @@ import collections import itertools - import asyncio import configparser @@ -97,7 +96,7 @@ def __init__(self, config=None, loop=None): self.commands = {} # Event runner - self.events = events.AsyncEventRunner(self._fire_hooks, self.loop) + self.events = events.HybridEventRunner(self._get_hooks, self.loop) # Keeps partial name lists between RPL_NAMREPLY and # RPL_ENDOFNAMES events @@ -113,9 +112,8 @@ def bot_teardown(self): """ self.plugins.teardown() - def _fire_hooks(self, event): - results = self.plugins.fire_hooks(event) - return list(itertools.chain(*results)) + def _get_hooks(self, event): + return itertools.chain(*self.plugins.get_hooks(event.event_type)) def post_event(self, event): return self.events.post_event(event) diff --git a/csbot/plugin.py b/csbot/plugin.py index 43eb7dd3..61124491 100644 --- a/csbot/plugin.py +++ b/csbot/plugin.py @@ -3,6 +3,7 @@ import logging import os import asyncio +from typing import List, Callable def build_plugin_dict(plugins): @@ -268,19 +269,10 @@ class Foo(Plugin): """ return ProvidedByPlugin(other, kwargs) - def fire_hooks(self, event): - """Execute all of this plugin's handlers for *event*. - - All handlers are treated as coroutine functions, and the return value is - a list of all the invoked coroutines. + def get_hooks(self, hook: str) -> List[Callable]: + """Get a list of this plugin's handlers for *hook*. """ - coros = [] - for name in self.plugin_hooks.get(event.event_type, ()): - f = getattr(self, name) - if not asyncio.iscoroutinefunction(f): - f = asyncio.coroutine(f) - coros.append(f(event)) - return coros + return [getattr(self, name) for name in self.plugin_hooks.get(hook, ())] def provide(self, plugin_name, **kwarg): """Provide a value for a :meth:`Plugin.use` usage.""" From 8de21c504ff81521650681e5050df1808f62983a Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Tue, 28 May 2019 21:22:51 +0100 Subject: [PATCH 19/66] Make LinkInfo.get_link_info() async, fix up tests --- csbot/plugins/linkinfo.py | 14 +++++++------- csbot/test/test_plugin_imgur.py | 15 +++++++++------ csbot/test/test_plugin_linkinfo.py | 26 +++++++++++++++----------- csbot/test/test_plugin_xkcd.py | 10 ++++++---- csbot/test/test_plugin_youtube.py | 5 +++-- 5 files changed, 40 insertions(+), 30 deletions(-) diff --git a/csbot/plugins/linkinfo.py b/csbot/plugins/linkinfo.py index 4e6c2014..a465112e 100644 --- a/csbot/plugins/linkinfo.py +++ b/csbot/plugins/linkinfo.py @@ -103,7 +103,7 @@ def register_exclude(self, filter): self.excludes.append(filter) @Plugin.command('link') - def link_command(self, e): + async def link_command(self, e): """Handle the "link" command. Fetch information about a specified URL, e.g. @@ -123,7 +123,7 @@ def link_command(self, e): url = 'http://' + url # Get info for the URL - result = self.get_link_info(url) + result = await self.get_link_info(url) self._log_if_error(result) # See if it was marked as NSFW in the command text result.nsfw |= 'nsfw' in rest.lower() @@ -131,7 +131,7 @@ def link_command(self, e): e.reply(result.get_message()) @Plugin.hook('core.message.privmsg') - def scan_privmsg(self, e): + async def scan_privmsg(self, e): """Scan the data of PRIVMSG events for URLs and respond with information about them. """ @@ -152,7 +152,7 @@ def scan_privmsg(self, e): break # Get info for the URL - result = self.get_link_info(part) + result = await self.get_link_info(part) self._log_if_error(result) if result.is_error: @@ -168,7 +168,7 @@ def scan_privmsg(self, e): # ... and since we got a useful result, stop processing the message break - def get_link_info(self, original_url): + async def get_link_info(self, original_url): """Get information about a URL. Using the *original_url* string, run the chain of URL handlers and @@ -205,11 +205,11 @@ def get_link_info(self, original_url): # Invoke the default handler if not excluded else: try: - return self.scrape_html_title(url) + return await self.scrape_html_title(url) except requests.exceptions.ConnectionError: return make_error('Connection error') - def scrape_html_title(self, url): + async def scrape_html_title(self, url): """Scrape the ```` tag contents from the HTML page at *url*. Returns a :class:`LinkInfoResult`. diff --git a/csbot/test/test_plugin_imgur.py b/csbot/test/test_plugin_imgur.py index dfe379fb..0903497d 100644 --- a/csbot/test/test_plugin_imgur.py +++ b/csbot/test/test_plugin_imgur.py @@ -172,12 +172,13 @@ def pre_irc_client(responses): content_type='application/json') +@pytest.mark.asyncio @pytest.mark.parametrize("url, api_url, status, content_type, fixture, title", test_cases) -def test_integration(bot_helper, responses, url, api_url, status, content_type, fixture, title): +async def test_integration(bot_helper, responses, url, api_url, status, content_type, fixture, title): responses.add(responses.GET, api_url, status=status, body=read_fixture_file(fixture), content_type=content_type) - result = bot_helper['linkinfo'].get_link_info(url) + result = await bot_helper['linkinfo'].get_link_info(url) if title is None: assert result.is_error else: @@ -185,12 +186,13 @@ def test_integration(bot_helper, responses, url, api_url, status, content_type, assert title == result.text +@pytest.mark.asyncio @pytest.mark.parametrize("url, api_url, status, content_type, fixture, title", nsfw_test_cases) -def test_integration_nsfw(bot_helper, responses, url, api_url, status, content_type, fixture, title): +async def test_integration_nsfw(bot_helper, responses, url, api_url, status, content_type, fixture, title): responses.add(responses.GET, api_url, status=status, body=read_fixture_file(fixture), content_type=content_type) - result = bot_helper['linkinfo'].get_link_info(url) + result = await bot_helper['linkinfo'].get_link_info(url) if title is None: assert result.is_error else: @@ -198,9 +200,10 @@ def test_integration_nsfw(bot_helper, responses, url, api_url, status, content_t assert title == result.text -def test_invalid_URL(bot_helper, responses): +@pytest.mark.asyncio +async def test_invalid_URL(bot_helper, responses): """Test that an unrecognised URL never even results in a request.""" responses.reset() # Drop requests used/made during plugin setup - result = bot_helper['linkinfo'].get_link_info('http://imgur.com/invalid/url') + result = await bot_helper['linkinfo'].get_link_info('http://imgur.com/invalid/url') assert result.is_error assert len(responses.calls) == 0 diff --git a/csbot/test/test_plugin_linkinfo.py b/csbot/test/test_plugin_linkinfo.py index 1d3a2c1a..dd658fb2 100644 --- a/csbot/test/test_plugin_linkinfo.py +++ b/csbot/test/test_plugin_linkinfo.py @@ -3,6 +3,7 @@ import unittest.mock as mock import pytest +import asynctest.mock import requests from csbot.util import simple_http_get @@ -138,38 +139,41 @@ async def irc_client(irc_client): return irc_client +@pytest.mark.asyncio @pytest.mark.parametrize("url, content_type, body, expected_title", encoding_test_cases, ids=[_[0] for _ in encoding_test_cases]) -def test_encoding_handling(bot_helper, responses, url, content_type, body, expected_title): +async def test_encoding_handling(bot_helper, responses, url, content_type, body, expected_title): responses.add(responses.GET, url, body=body, content_type=content_type, stream=True) - result = bot_helper['linkinfo'].get_link_info(url) + result = await bot_helper['linkinfo'].get_link_info(url) assert result.text == expected_title +@pytest.mark.asyncio @pytest.mark.parametrize("url, content_type, body", error_test_cases, ids=[_[0] for _ in error_test_cases]) -def test_html_title_errors(bot_helper, responses, url, content_type, body): +async def test_html_title_errors(bot_helper, responses, url, content_type, body): responses.add(responses.GET, url, body=body, content_type=content_type, stream=True) - result = bot_helper['linkinfo'].get_link_info(url) + result = await bot_helper['linkinfo'].get_link_info(url) assert result.is_error -def test_connection_error(bot_helper, responses): +@pytest.mark.asyncio +async def test_connection_error(bot_helper, responses): # Check our assumptions: should be connection error because "responses" library is mocking the internet with pytest.raises(requests.ConnectionError): simple_http_get('http://example.com/foo/bar') # Should result in an error message from linkinfo (and implicitly no exception raised) - result = bot_helper['linkinfo'].get_link_info('http://example.com/foo/bar') + result = await bot_helper['linkinfo'].get_link_info('http://example.com/foo/bar') assert result.is_error @pytest.mark.usefixtures("run_client") @pytest.mark.asyncio @pytest.mark.parametrize("msg, urls", [('http://example.com', ['http://example.com'])]) -def test_scan_privmsg(bot_helper, msg, urls): - with mock.patch.object(bot_helper['linkinfo'], 'get_link_info') as get_link_info: - yield from bot_helper.client.line_received(':nick!user@host PRIVMSG #channel :' + msg) +async def test_scan_privmsg(event_loop, bot_helper, msg, urls): + with asynctest.mock.patch.object(bot_helper['linkinfo'], 'get_link_info') as get_link_info: + await bot_helper.client.line_received(':nick!user@host PRIVMSG #channel :' + msg) get_link_info.assert_has_calls([mock.call(url) for url in urls]) @@ -185,10 +189,10 @@ def test_scan_privmsg_rate_limit(bot_helper): linkinfo = bot_helper['linkinfo'] count = int(linkinfo.config_get('rate_limit_count')) for i in range(count): - with mock.patch.object(linkinfo, 'get_link_info') as get_link_info: + with asynctest.mock.patch.object(linkinfo, 'get_link_info', ) as get_link_info: yield from bot_helper.client.line_received( ':nick!user@host PRIVMSG #channel :http://example.com/{}'.format(i)) get_link_info.assert_called_once_with('http://example.com/{}'.format(i)) - with mock.patch.object(linkinfo, 'get_link_info') as get_link_info: + with asynctest.mock.patch.object(linkinfo, 'get_link_info') as get_link_info: yield from bot_helper.client.line_received(':nick!user@host PRIVMSG #channel :http://example.com/12345') assert not get_link_info.called diff --git a/csbot/test/test_plugin_xkcd.py b/csbot/test/test_plugin_xkcd.py index 844c647a..c71b3a86 100644 --- a/csbot/test/test_plugin_xkcd.py +++ b/csbot/test/test_plugin_xkcd.py @@ -156,17 +156,19 @@ def populate_responses(self, bot_helper, responses): content_type=content_type) @pytest.mark.usefixtures("populate_responses") + @pytest.mark.asyncio @pytest.mark.parametrize("num, url, content_type, fixture, expected", json_test_cases, ids=[_[1] for _ in json_test_cases]) - def test_integration(self, bot_helper, num, url, content_type, fixture, expected): + async def test_integration(self, bot_helper, num, url, content_type, fixture, expected): _, title, alt = expected url = 'http://xkcd.com/{}'.format(num) - result = bot_helper['linkinfo'].get_link_info(url) + result = await bot_helper['linkinfo'].get_link_info(url) assert title in result.text assert alt in result.text @pytest.mark.usefixtures("populate_responses") - def test_integration_error(self, bot_helper): + @pytest.mark.asyncio + async def test_integration_error(self, bot_helper): # Error case - result = bot_helper['linkinfo'].get_link_info("http://xkcd.com/flibble") + result = await bot_helper['linkinfo'].get_link_info("http://xkcd.com/flibble") assert result.is_error diff --git a/csbot/test/test_plugin_youtube.py b/csbot/test/test_plugin_youtube.py index 9552fd68..c2178e87 100644 --- a/csbot/test/test_plugin_youtube.py +++ b/csbot/test/test_plugin_youtube.py @@ -125,6 +125,7 @@ def bot_helper(self, bot_helper): }) return bot_helper + @pytest.mark.asyncio @pytest.mark.parametrize("vid_id, status, fixture, response", json_test_cases) @pytest.mark.parametrize("url", [ "https://www.youtube.com/watch?v={}", @@ -133,11 +134,11 @@ def bot_helper(self, bot_helper): "http://www.youtube.com/watch?v={}&feature=youtube_gdata_player", "http://youtu.be/{}", ]) - def test_integration(self, bot_helper, vid_id, status, fixture, response, url): + async def test_integration(self, bot_helper, vid_id, status, fixture, response, url): http = apiclient.http.HttpMock(fixture_file(fixture), {'status': status}) with patch.object(bot_helper['youtube'], 'http', wraps=http): url = url.format(vid_id) - result = bot_helper['linkinfo'].get_link_info(url) + result = await bot_helper['linkinfo'].get_link_info(url) if response is None or response is YoutubeError: assert result.is_error else: From 11315cff821bf481d846a0be5a4f7040d93aa644 Mon Sep 17 00:00:00 2001 From: Alan Briolat <alan.briolat@gmail.com> Date: Wed, 29 May 2019 12:02:32 +0100 Subject: [PATCH 20/66] Use aiohttp instead of requests, for async LinkInfo behaviour --- csbot/plugins/linkinfo.py | 18 +++++++-------- csbot/test/test_plugin_linkinfo.py | 35 +++++++++++++++++------------- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/csbot/plugins/linkinfo.py b/csbot/plugins/linkinfo.py index a465112e..2a6bbd25 100644 --- a/csbot/plugins/linkinfo.py +++ b/csbot/plugins/linkinfo.py @@ -5,14 +5,14 @@ from collections import namedtuple import datetime from functools import partial -from contextlib import closing import requests +import aiohttp import lxml.etree import lxml.html from ..plugin import Plugin -from ..util import simple_http_get, Struct +from ..util import Struct LinkInfoHandler = namedtuple('LinkInfoHandler', ['filter', 'handler', 'exclusive']) @@ -217,11 +217,11 @@ async def scrape_html_title(self, url): make_error = partial(LinkInfoResult, url.geturl(), is_error=True) # Let's see what's on the other end... - with closing(simple_http_get(url.geturl(), stream=True)) as r: + async with aiohttp.ClientSession() as session, session.get(url.geturl()) as r: # Only bother with 200 OK - if r.status_code != requests.codes.ok: - return make_error('HTTP request failed: {}' - .format(r.status_code)) + if r.status != 200: + return make_error('HTTP request failed: {} {}' + .format(r.status, r.reason)) # Only process HTML-ish responses if 'Content-Type' not in r.headers: return make_error('No Content-Type header') @@ -240,8 +240,8 @@ async def scrape_html_title(self, url): # If present, charset attribute in HTTP Content-Type header takes # precedence, but fallback to default if encoding isn't recognised parser = lxml.html.html_parser - if 'charset=' in r.headers['content-type']: - encoding = r.headers['content-type'].rsplit('=', 1)[1] + if r.charset is not None: + encoding = r.charset try: parser = lxml.html.HTMLParser(encoding=encoding) except LookupError: @@ -252,7 +252,7 @@ async def scrape_html_title(self, url): # because chunk-encoded responses iterate over chunks rather than # the size we request... chunk = b'' - for next_chunk in r.iter_content(self.config_get('max_response_size')): + async for next_chunk in r.content.iter_chunked(self.config_get('max_response_size')): chunk += next_chunk if len(chunk) >= self.config_get('max_response_size'): break diff --git a/csbot/test/test_plugin_linkinfo.py b/csbot/test/test_plugin_linkinfo.py index dd658fb2..bc8c7ce2 100644 --- a/csbot/test/test_plugin_linkinfo.py +++ b/csbot/test/test_plugin_linkinfo.py @@ -1,12 +1,11 @@ # coding=utf-8 from lxml.etree import LIBXML_VERSION import unittest.mock as mock +import urllib.parse as urlparse import pytest import asynctest.mock -import requests - -from csbot.util import simple_http_get +import aiohttp #: Test encoding handling; tests are (url, content-type, body, expected_title) @@ -142,8 +141,10 @@ async def irc_client(irc_client): @pytest.mark.asyncio @pytest.mark.parametrize("url, content_type, body, expected_title", encoding_test_cases, ids=[_[0] for _ in encoding_test_cases]) -async def test_encoding_handling(bot_helper, responses, url, content_type, body, expected_title): - responses.add(responses.GET, url, body=body, content_type=content_type, stream=True) +async def test_encoding_handling(bot_helper, aresponses, url, content_type, body, expected_title): + parsed_url = urlparse.urlparse(url) + aresponses.add(parsed_url.hostname, parsed_url.path, 'get', + aresponses.Response(body=body, headers={'Content-Type': content_type})) result = await bot_helper['linkinfo'].get_link_info(url) assert result.text == expected_title @@ -151,27 +152,31 @@ async def test_encoding_handling(bot_helper, responses, url, content_type, body, @pytest.mark.asyncio @pytest.mark.parametrize("url, content_type, body", error_test_cases, ids=[_[0] for _ in error_test_cases]) -async def test_html_title_errors(bot_helper, responses, url, content_type, body): - responses.add(responses.GET, url, body=body, - content_type=content_type, stream=True) +async def test_html_title_errors(bot_helper, aresponses, url, content_type, body): + parsed_url = urlparse.urlparse(url) + aresponses.add(parsed_url.hostname, parsed_url.path, 'get', + aresponses.Response(body=body, headers={'Content-Type': content_type})) result = await bot_helper['linkinfo'].get_link_info(url) assert result.is_error @pytest.mark.asyncio -async def test_connection_error(bot_helper, responses): - # Check our assumptions: should be connection error because "responses" library is mocking the internet - with pytest.raises(requests.ConnectionError): - simple_http_get('http://example.com/foo/bar') +async def test_not_found(bot_helper, aresponses): + # Test our assumptions: direct request should return a 404 + aresponses.add(aresponses.ANY, aresponses.ANY, aresponses.ANY, lambda r: aresponses.Response(status=404)) + async with aiohttp.ClientSession() as session, session.get('http://example.com/') as resp: + assert resp.status == 404 + # Should result in an error message from linkinfo (and implicitly no exception raised) - result = await bot_helper['linkinfo'].get_link_info('http://example.com/foo/bar') + aresponses.add(aresponses.ANY, aresponses.ANY, aresponses.ANY, lambda r: aresponses.Response(status=404)) + result = await bot_helper['linkinfo'].get_link_info('http://example.com/') assert result.is_error @pytest.mark.usefixtures("run_client") @pytest.mark.asyncio @pytest.mark.parametrize("msg, urls", [('http://example.com', ['http://example.com'])]) -async def test_scan_privmsg(event_loop, bot_helper, msg, urls): +async def test_scan_privmsg(event_loop, bot_helper, aresponses, msg, urls): with asynctest.mock.patch.object(bot_helper['linkinfo'], 'get_link_info') as get_link_info: await bot_helper.client.line_received(':nick!user@host PRIVMSG #channel :' + msg) get_link_info.assert_has_calls([mock.call(url) for url in urls]) @@ -179,7 +184,7 @@ async def test_scan_privmsg(event_loop, bot_helper, msg, urls): @pytest.mark.usefixtures("run_client") @pytest.mark.asyncio -def test_scan_privmsg_rate_limit(bot_helper): +def test_scan_privmsg_rate_limit(bot_helper, aresponses): """Test that we won't respond too frequently to URLs in messages. Unfortunately we can't currently test the passage of time, so the only From 5d680916d877489c0b95d569e8714bf3fbc5d81a Mon Sep 17 00:00:00 2001 From: Alan Briolat <alan.briolat@gmail.com> Date: Wed, 29 May 2019 17:56:57 +0100 Subject: [PATCH 21/66] Test non-blocking behaviour of linkinfo plugin --- csbot/events.py | 3 +- csbot/test/test_plugin_linkinfo.py | 112 ++++++++++++++++++++++++++++- 2 files changed, 112 insertions(+), 3 deletions(-) diff --git a/csbot/events.py b/csbot/events.py index 61a492e9..4f4f8d06 100644 --- a/csbot/events.py +++ b/csbot/events.py @@ -8,7 +8,6 @@ LOG = logging.getLogger('csbot.events') -LOG.setLevel(logging.INFO) class ImmediateEventRunner(object): @@ -190,9 +189,11 @@ def _run_events(self): LOG.debug('processing events (%s remaining)', len(self.events)) # Get next event event = self.events.popleft() + LOG.debug('processing event: %s', event) # Handle the event for handler in self.get_handlers(event): # Attempt to run the handler, but don't break everything if the handler fails + LOG.debug('running handler: %r', handler) try: result = handler(event) except Exception as e: diff --git a/csbot/test/test_plugin_linkinfo.py b/csbot/test/test_plugin_linkinfo.py index bc8c7ce2..c8cb2557 100644 --- a/csbot/test/test_plugin_linkinfo.py +++ b/csbot/test/test_plugin_linkinfo.py @@ -2,11 +2,15 @@ from lxml.etree import LIBXML_VERSION import unittest.mock as mock import urllib.parse as urlparse +import asyncio import pytest import asynctest.mock import aiohttp +from csbot.plugin import Plugin +import csbot.core + #: Test encoding handling; tests are (url, content-type, body, expected_title) encoding_test_cases = [ @@ -173,7 +177,6 @@ async def test_not_found(bot_helper, aresponses): assert result.is_error -@pytest.mark.usefixtures("run_client") @pytest.mark.asyncio @pytest.mark.parametrize("msg, urls", [('http://example.com', ['http://example.com'])]) async def test_scan_privmsg(event_loop, bot_helper, aresponses, msg, urls): @@ -182,7 +185,17 @@ async def test_scan_privmsg(event_loop, bot_helper, aresponses, msg, urls): get_link_info.assert_has_calls([mock.call(url) for url in urls]) -@pytest.mark.usefixtures("run_client") +@pytest.mark.asyncio +@pytest.mark.parametrize("msg, urls", [('http://example.com', ['http://example.com'])]) +async def test_command(event_loop, bot_helper, aresponses, msg, urls): + with asynctest.mock.patch.object(bot_helper['linkinfo'], 'get_link_info') as get_link_info, \ + asynctest.mock.patch.object(bot_helper['linkinfo'], 'link_command', + wraps=bot_helper['linkinfo'].link_command) as link_command: + await bot_helper.client.line_received(':nick!user@host PRIVMSG #channel :!link ' + msg) + get_link_info.assert_has_calls([mock.call(url) for url in urls]) + assert link_command.call_count == 1 + + @pytest.mark.asyncio def test_scan_privmsg_rate_limit(bot_helper, aresponses): """Test that we won't respond too frequently to URLs in messages. @@ -201,3 +214,98 @@ def test_scan_privmsg_rate_limit(bot_helper, aresponses): with asynctest.mock.patch.object(linkinfo, 'get_link_info') as get_link_info: yield from bot_helper.client.line_received(':nick!user@host PRIVMSG #channel :http://example.com/12345') assert not get_link_info.called + + +class MockPlugin(Plugin): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.handler_mock = mock.Mock(spec=callable) + + @Plugin.hook('core.message.privmsg') + def privmsg(self, event): + self.handler_mock(event['message']) + + +class TestNonBlocking: + class Bot(csbot.core.Bot): + available_plugins = csbot.core.Bot.available_plugins.copy() + available_plugins.update(mockplugin=MockPlugin) + + # TODO: this is ugly, need to improve (a) subclassing behaviour and/or (b) test utilities + privmsg = Plugin.hook('core.message.privmsg')(csbot.core.Bot.privmsg) + fire_command = Plugin.hook('core.command')(csbot.core.Bot.fire_command) + + CONFIG = f"""\ + [@bot] + plugins = mockplugin linkinfo + """ + + pytestmark = pytest.mark.bot(cls=Bot, config=CONFIG) + + @pytest.mark.asyncio + async def test_non_blocking_privmsg(self, event_loop, bot_helper, aresponses): + bot_helper.reset_mock() + + event = asyncio.Event(loop=event_loop) + + async def handler(request): + await event.wait() + return aresponses.Response(status=200, + headers={'Content-Type': 'text/html'}, + body=b'<html><head><title>foo') + aresponses.add('example.com', '/', 'get', handler) + + futures = bot_helper.receive([ + ':nick!user@host PRIVMSG #channel :a', + ':nick!user@host PRIVMSG #channel :http://example.com/', + ':nick!user@host PRIVMSG #channel :b', + ]) + await asyncio.wait(futures, loop=event_loop, timeout=0.1) + assert bot_helper['mockplugin'].handler_mock.mock_calls == [ + mock.call('a'), + mock.call('http://example.com/'), + mock.call('b'), + ] + bot_helper.client.send_line.assert_not_called() + + event.set() + await asyncio.wait(futures, loop=event_loop, timeout=0.1) + assert all(f.done() for f in futures) + bot_helper.client.send_line.assert_has_calls([ + mock.call('NOTICE #channel :foo'), + ]) + + @pytest.mark.asyncio + async def test_non_blocking_command(self, event_loop, bot_helper, aresponses): + bot_helper.reset_mock() + + event = asyncio.Event(loop=event_loop) + + async def handler(request): + await event.wait() + return aresponses.Response(status=200, + headers={'Content-Type': 'application/octet-stream'}, + body=b'foo') + + aresponses.add('example.com', '/', 'get', handler) + + futures = bot_helper.receive([ + ':nick!user@host PRIVMSG #channel :a', + ':nick!user@host PRIVMSG #channel :!link http://example.com/', + ':nick!user@host PRIVMSG #channel :b', + ]) + await asyncio.wait(futures, loop=event_loop, timeout=0.1) + assert bot_helper['mockplugin'].handler_mock.mock_calls == [ + mock.call('a'), + mock.call('!link http://example.com/'), + mock.call('b'), + ] + bot_helper.client.send_line.assert_not_called() + + event.set() + await asyncio.wait(futures, loop=event_loop, timeout=0.1) + assert all(f.done() for f in futures) + bot_helper.client.send_line.assert_has_calls([ + mock.call('NOTICE #channel :Error: Content-Type not HTML-ish: ' + 'application/octet-stream (http://example.com/)'), + ]) From 867058b7a40a8540b30db24567c5932a6cecc11d Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Wed, 29 May 2019 20:25:37 +0100 Subject: [PATCH 22/66] Oops, forgot aresponses dependency... --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index dd399f48..4d6804da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,6 +15,7 @@ rollbar pytest==4.2.0 pytest-asyncio==0.10.0 pytest-aiohttp==0.3.0 +aresponses==1.1.1 pytest-cov==2.6.1 asynctest==0.12.2 aiofastforward==0.0.17 From 51194a866232a0d6d9356ff12f55c5e0dc459c02 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Wed, 29 May 2019 21:03:41 +0100 Subject: [PATCH 23/66] Use aioresponses instead of aresponses (where possible) --- csbot/plugins/linkinfo.py | 2 +- csbot/test/conftest.py | 7 +++++++ csbot/test/test_plugin_linkinfo.py | 33 ++++++++++++++---------------- requirements.txt | 1 + 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/csbot/plugins/linkinfo.py b/csbot/plugins/linkinfo.py index 2a6bbd25..41121d57 100644 --- a/csbot/plugins/linkinfo.py +++ b/csbot/plugins/linkinfo.py @@ -206,7 +206,7 @@ async def get_link_info(self, original_url): else: try: return await self.scrape_html_title(url) - except requests.exceptions.ConnectionError: + except aiohttp.ClientConnectionError: return make_error('Connection error') async def scrape_html_title(self, url): diff --git a/csbot/test/conftest.py b/csbot/test/conftest.py index 048c4dc1..145e073f 100644 --- a/csbot/test/conftest.py +++ b/csbot/test/conftest.py @@ -6,6 +6,7 @@ import pytest import aiofastforward import responses as responses_ +from aioresponses import aioresponses as aioresponses_ from csbot import test from csbot.irc import IRCClient @@ -176,3 +177,9 @@ def bot(self): def responses(): with responses_.RequestsMock() as rsps: yield rsps + + +@pytest.fixture +def aioresponses(): + with aioresponses_() as m: + yield m diff --git a/csbot/test/test_plugin_linkinfo.py b/csbot/test/test_plugin_linkinfo.py index c8cb2557..0ed6c9f6 100644 --- a/csbot/test/test_plugin_linkinfo.py +++ b/csbot/test/test_plugin_linkinfo.py @@ -1,7 +1,6 @@ # coding=utf-8 from lxml.etree import LIBXML_VERSION import unittest.mock as mock -import urllib.parse as urlparse import asyncio import pytest @@ -145,10 +144,8 @@ async def irc_client(irc_client): @pytest.mark.asyncio @pytest.mark.parametrize("url, content_type, body, expected_title", encoding_test_cases, ids=[_[0] for _ in encoding_test_cases]) -async def test_encoding_handling(bot_helper, aresponses, url, content_type, body, expected_title): - parsed_url = urlparse.urlparse(url) - aresponses.add(parsed_url.hostname, parsed_url.path, 'get', - aresponses.Response(body=body, headers={'Content-Type': content_type})) +async def test_encoding_handling(bot_helper, aioresponses, url, content_type, body, expected_title): + aioresponses.get(url, status=200, body=body, headers={'Content-Type': content_type}) result = await bot_helper['linkinfo'].get_link_info(url) assert result.text == expected_title @@ -156,30 +153,28 @@ async def test_encoding_handling(bot_helper, aresponses, url, content_type, body @pytest.mark.asyncio @pytest.mark.parametrize("url, content_type, body", error_test_cases, ids=[_[0] for _ in error_test_cases]) -async def test_html_title_errors(bot_helper, aresponses, url, content_type, body): - parsed_url = urlparse.urlparse(url) - aresponses.add(parsed_url.hostname, parsed_url.path, 'get', - aresponses.Response(body=body, headers={'Content-Type': content_type})) +async def test_html_title_errors(bot_helper, aioresponses, url, content_type, body): + aioresponses.get(url, status=200, body=body, headers={'Content-Type': content_type}) result = await bot_helper['linkinfo'].get_link_info(url) assert result.is_error @pytest.mark.asyncio -async def test_not_found(bot_helper, aresponses): - # Test our assumptions: direct request should return a 404 - aresponses.add(aresponses.ANY, aresponses.ANY, aresponses.ANY, lambda r: aresponses.Response(status=404)) - async with aiohttp.ClientSession() as session, session.get('http://example.com/') as resp: - assert resp.status == 404 +async def test_not_found(bot_helper, aioresponses): + # Test our assumptions: direct request should raise connection error, because aioresponses + # is mocking the internet + with pytest.raises(aiohttp.ClientConnectionError): + async with aiohttp.ClientSession() as session, session.get('http://example.com/') as resp: + pass # Should result in an error message from linkinfo (and implicitly no exception raised) - aresponses.add(aresponses.ANY, aresponses.ANY, aresponses.ANY, lambda r: aresponses.Response(status=404)) result = await bot_helper['linkinfo'].get_link_info('http://example.com/') assert result.is_error @pytest.mark.asyncio @pytest.mark.parametrize("msg, urls", [('http://example.com', ['http://example.com'])]) -async def test_scan_privmsg(event_loop, bot_helper, aresponses, msg, urls): +async def test_scan_privmsg(event_loop, bot_helper, aioresponses, msg, urls): with asynctest.mock.patch.object(bot_helper['linkinfo'], 'get_link_info') as get_link_info: await bot_helper.client.line_received(':nick!user@host PRIVMSG #channel :' + msg) get_link_info.assert_has_calls([mock.call(url) for url in urls]) @@ -187,7 +182,7 @@ async def test_scan_privmsg(event_loop, bot_helper, aresponses, msg, urls): @pytest.mark.asyncio @pytest.mark.parametrize("msg, urls", [('http://example.com', ['http://example.com'])]) -async def test_command(event_loop, bot_helper, aresponses, msg, urls): +async def test_command(event_loop, bot_helper, aioresponses, msg, urls): with asynctest.mock.patch.object(bot_helper['linkinfo'], 'get_link_info') as get_link_info, \ asynctest.mock.patch.object(bot_helper['linkinfo'], 'link_command', wraps=bot_helper['linkinfo'].link_command) as link_command: @@ -197,7 +192,7 @@ async def test_command(event_loop, bot_helper, aresponses, msg, urls): @pytest.mark.asyncio -def test_scan_privmsg_rate_limit(bot_helper, aresponses): +def test_scan_privmsg_rate_limit(bot_helper, aioresponses): """Test that we won't respond too frequently to URLs in messages. Unfortunately we can't currently test the passage of time, so the only @@ -244,6 +239,7 @@ class Bot(csbot.core.Bot): @pytest.mark.asyncio async def test_non_blocking_privmsg(self, event_loop, bot_helper, aresponses): + # TODO: use aioresponses instead, once it supports async callbacks bot_helper.reset_mock() event = asyncio.Event(loop=event_loop) @@ -277,6 +273,7 @@ async def handler(request): @pytest.mark.asyncio async def test_non_blocking_command(self, event_loop, bot_helper, aresponses): + # TODO: use aioresponses instead, once it supports async callbacks bot_helper.reset_mock() event = asyncio.Event(loop=event_loop) diff --git a/requirements.txt b/requirements.txt index 4d6804da..096a35ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,6 +16,7 @@ pytest==4.2.0 pytest-asyncio==0.10.0 pytest-aiohttp==0.3.0 aresponses==1.1.1 +aioresponses==0.6.0 pytest-cov==2.6.1 asynctest==0.12.2 aiofastforward==0.0.17 From da1ba367f2b3f9f3394983b18f628a2b2f28ba49 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Wed, 29 May 2019 21:17:46 +0100 Subject: [PATCH 24/66] Simplify HTTP request code with simple_http_get_async --- csbot/plugins/linkinfo.py | 4 ++-- csbot/util.py | 17 +++++++++++++++++ requirements.txt | 1 + 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/csbot/plugins/linkinfo.py b/csbot/plugins/linkinfo.py index 41121d57..c92f3e1a 100644 --- a/csbot/plugins/linkinfo.py +++ b/csbot/plugins/linkinfo.py @@ -12,7 +12,7 @@ import lxml.html from ..plugin import Plugin -from ..util import Struct +from ..util import Struct, simple_http_get_async LinkInfoHandler = namedtuple('LinkInfoHandler', ['filter', 'handler', 'exclusive']) @@ -217,7 +217,7 @@ async def scrape_html_title(self, url): make_error = partial(LinkInfoResult, url.geturl(), is_error=True) # Let's see what's on the other end... - async with aiohttp.ClientSession() as session, session.get(url.geturl()) as r: + async with simple_http_get_async(url.geturl()) as r: # Only bother with 200 OK if r.status != 200: return make_error('HTTP request failed: {} {}' diff --git a/csbot/util.py b/csbot/util.py index 7022657f..214b214e 100644 --- a/csbot/util.py +++ b/csbot/util.py @@ -3,6 +3,8 @@ from collections import OrderedDict import requests +from async_generator import asynccontextmanager +import aiohttp class User(object): @@ -96,6 +98,21 @@ def simple_http_get(url, stream=False): return requests.get(url, verify=False, headers=headers, stream=stream) +@asynccontextmanager +async def simple_http_get_async(url): + session_kwargs = { + 'headers': { + 'User-Agent': 'csbot/0.1', + }, + } + get_kwargs = { + 'ssl': False, + } + async with aiohttp.ClientSession(**session_kwargs) as session: + async with session.get(url, **get_kwargs) as resp: + yield resp + + def pairwise(iterable): """Pairs elements of an iterable together, e.g. s -> (s0,s1), (s1,s2), (s2, s3), ... diff --git a/requirements.txt b/requirements.txt index 096a35ee..8f515f61 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,7 @@ oauth2client>=3,<4 # google-api-python-client dropped dep in a minor releas imgurpython>=1.1.6,<2.0.0 isodate>=0.5.1 aiohttp>=3.5.1,<4.0 +async_generator>=1.10,<2.0 rollbar # Requirements for unit testing From 4b1ab7e428161736a62e3f9e94d1fd5b5e544219 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Wed, 29 May 2019 21:38:02 +0100 Subject: [PATCH 25/66] Make XKCD plugin async, update tests LinkInfo integration doesn't support async handlers (yet) --- csbot/plugins/xkcd.py | 31 ++++++++++----------- csbot/test/test_plugin_xkcd.py | 51 +++++++++++++++++++--------------- 2 files changed, 44 insertions(+), 38 deletions(-) diff --git a/csbot/plugins/xkcd.py b/csbot/plugins/xkcd.py index 909fccdd..29ef5002 100644 --- a/csbot/plugins/xkcd.py +++ b/csbot/plugins/xkcd.py @@ -1,9 +1,8 @@ import html import random -import requests from ..plugin import Plugin -from ..util import simple_http_get, cap_string, is_ascii +from ..util import simple_http_get_async, cap_string, is_ascii from .linkinfo import LinkInfoResult @@ -29,7 +28,7 @@ def fix_json_unicode(data): return data -def get_info(number=None): +async def get_info(number=None): """Gets the json data for a particular comic (or the latest, if none provided). """ @@ -38,13 +37,13 @@ def get_info(number=None): else: url = "http://xkcd.com/info.0.json" - httpdata = simple_http_get(url) - if httpdata.status_code != requests.codes.ok: - return None + async with simple_http_get_async(url) as httpdata: + if httpdata.status != 200: + return None - # Only care about part of the data - httpjson = httpdata.json() - data = {key: httpjson[key] for key in ["title", "alt", "num"]} + # Only care about part of the data + httpjson = await httpdata.json() + data = {key: httpjson[key] for key in ["title", "alt", "num"]} # Unfuck up unicode strings data = fix_json_unicode(data) @@ -61,12 +60,12 @@ class xkcd(Plugin): class XKCDError(Exception): pass - def _xkcd(self, user_str): + async def _xkcd(self, user_str): """Get the url and title stuff. Returns a string of the response. """ - latest = get_info() + latest = await get_info() if not latest: raise self.XKCDError("Error getting comics") @@ -75,12 +74,12 @@ def _xkcd(self, user_str): if not user_str or user_str in {'0', 'latest', 'current', 'newest'}: requested = latest elif user_str in {'rand', 'random'}: - requested = get_info(random.randint(1, latest_num)) + requested = await get_info(random.randint(1, latest_num)) else: try: num = int(user_str) if 1 <= num <= latest_num: - requested = get_info(num) + requested = await get_info(num) else: raise self.XKCDError("Comic #{} is invalid. The latest is #{}" .format(num, latest_num)) @@ -95,18 +94,18 @@ def _xkcd(self, user_str): return (requested["url"], requested["title"], cap_string(requested["alt"], 120)) - @Plugin.integrate_with('linkinfo') + # @Plugin.integrate_with('linkinfo') def linkinfo_integrate(self, linkinfo): """Handle recognised xkcd urls.""" - def page_handler(url, match): + async def page_handler(url, match): """Use the main _xkcd function, then modify the result (if success) so it looks nicer. """ # Remove leading and trailing '/' try: - response = self._xkcd(url.path.strip('/')) + response = await self._xkcd(url.path.strip('/')) return LinkInfoResult(url.geturl(), '{1} - "{2}"'.format(*response)) except self.XKCDError: return None diff --git a/csbot/test/test_plugin_xkcd.py b/csbot/test/test_plugin_xkcd.py index c71b3a86..c7ca6cdf 100644 --- a/csbot/test/test_plugin_xkcd.py +++ b/csbot/test/test_plugin_xkcd.py @@ -87,48 +87,54 @@ """) class TestXKCDPlugin: @pytest.fixture - def populate_responses(self, bot_helper, responses): - """Populate all data into responses, don't assert that every request is fired.""" - responses.assert_all_requests_are_fired = False + def populate_responses(self, bot_helper, aioresponses): + """Populate all data into responses.""" for num, url, content_type, fixture, expected in json_test_cases: - responses.add(responses.GET, url, body=read_fixture_file(fixture), - content_type=content_type) + aioresponses.add(url, body=read_fixture_file(fixture), + content_type=content_type) + @pytest.mark.asyncio @pytest.mark.usefixtures("populate_responses") @pytest.mark.parametrize("num, url, content_type, fixture, expected", json_test_cases, ids=[_[1] for _ in json_test_cases]) - def test_correct(self, bot_helper, num, url, content_type, fixture, expected): - result = bot_helper['xkcd']._xkcd(num) + async def test_correct(self, bot_helper, num, url, content_type, fixture, expected): + result = await bot_helper['xkcd']._xkcd(num) assert result == expected + @pytest.mark.asyncio @pytest.mark.usefixtures("populate_responses") - def test_latest_success(self, bot_helper): + async def test_latest_success(self, bot_helper): # Also test the empty string num, url, content_type, fixture, expected = json_test_cases[0] - assert bot_helper['xkcd']._xkcd("") == expected + assert await bot_helper['xkcd']._xkcd("") == expected + @pytest.mark.asyncio @pytest.mark.usefixtures("populate_responses") - def test_random(self, bot_helper): + async def test_random(self, bot_helper): # !xkcd 221 num, url, content_type, fixture, expected = json_test_cases[1] with patch("random.randint", return_value=1): - assert bot_helper['xkcd']._xkcd("rand") == expected + assert await bot_helper['xkcd']._xkcd("rand") == expected - def test_error(self, bot_helper, responses): + @pytest.mark.asyncio + async def test_error_1(self, bot_helper, aioresponses): num, url, content_type, fixture, _ = json_test_cases[0] # Latest # Test if the comics are unavailable by making the latest return a 404 - responses.add(responses.GET, url, body="404 - Not Found", - content_type="text/html", status=404) + aioresponses.get(url, body="404 - Not Found", + content_type="text/html", status=404) with pytest.raises(bot_helper['xkcd'].XKCDError): - bot_helper['xkcd']._xkcd("") - responses.reset() + await bot_helper['xkcd']._xkcd("") + @pytest.mark.asyncio + async def test_error_2(self, bot_helper, aioresponses): + num, url, content_type, fixture, _ = json_test_cases[0] # Latest # Now override the actual 404 page and the latest "properly" - responses.add(responses.GET, url, body=read_fixture_file(fixture), - content_type=content_type) - responses.add(responses.GET, "http://xkcd.com/404/info.0.json", - body="404 - Not Found", content_type="text/html", - status=404) + aioresponses.get(url, body=read_fixture_file(fixture), + content_type=content_type, + repeat=True) + aioresponses.get("http://xkcd.com/404/info.0.json", + body="404 - Not Found", content_type="text/html", + status=404) error_cases = [ "flibble", @@ -139,9 +145,10 @@ def test_error(self, bot_helper, responses): for case in error_cases: with pytest.raises(bot_helper['xkcd'].XKCDError): - bot_helper['xkcd']._xkcd(case) + await bot_helper['xkcd']._xkcd(case) +@pytest.mark.skip @pytest.mark.bot(config="""\ [@bot] plugins = linkinfo xkcd From 45ee669b58e3c97382cce7d73d412d4eea14c7bb Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Wed, 29 May 2019 21:56:25 +0100 Subject: [PATCH 26/66] Support async LinkInfo integration handlers --- csbot/plugins/linkinfo.py | 7 +++++-- csbot/plugins/xkcd.py | 2 +- csbot/test/test_plugin_xkcd.py | 10 ++++------ 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/csbot/plugins/linkinfo.py b/csbot/plugins/linkinfo.py index c92f3e1a..d8b948da 100644 --- a/csbot/plugins/linkinfo.py +++ b/csbot/plugins/linkinfo.py @@ -5,8 +5,8 @@ from collections import namedtuple import datetime from functools import partial +import asyncio -import requests import aiohttp import lxml.etree import lxml.html @@ -186,7 +186,10 @@ async def get_link_info(self, original_url): for h in self.handlers: match = h.filter(url) if match: - result = h.handler(url, match) + if asyncio.iscoroutinefunction(h.handler): + result = await h.handler(url, match) + else: + result = h.handler(url, match) if result is not None: # Useful result, return it return result diff --git a/csbot/plugins/xkcd.py b/csbot/plugins/xkcd.py index 29ef5002..d4b847c9 100644 --- a/csbot/plugins/xkcd.py +++ b/csbot/plugins/xkcd.py @@ -94,7 +94,7 @@ async def _xkcd(self, user_str): return (requested["url"], requested["title"], cap_string(requested["alt"], 120)) - # @Plugin.integrate_with('linkinfo') + @Plugin.integrate_with('linkinfo') def linkinfo_integrate(self, linkinfo): """Handle recognised xkcd urls.""" diff --git a/csbot/test/test_plugin_xkcd.py b/csbot/test/test_plugin_xkcd.py index c7ca6cdf..4863a340 100644 --- a/csbot/test/test_plugin_xkcd.py +++ b/csbot/test/test_plugin_xkcd.py @@ -148,19 +148,17 @@ async def test_error_2(self, bot_helper, aioresponses): await bot_helper['xkcd']._xkcd(case) -@pytest.mark.skip @pytest.mark.bot(config="""\ [@bot] plugins = linkinfo xkcd """) class TestXKCDLinkInfoIntegration: @pytest.fixture - def populate_responses(self, bot_helper, responses): - """Populate all data into responses, don't assert that every request is fired.""" - responses.assert_all_requests_are_fired = False + def populate_responses(self, bot_helper, aioresponses): + """Populate all data into responses.""" for num, url, content_type, fixture, expected in json_test_cases: - responses.add(responses.GET, url, body=read_fixture_file(fixture), - content_type=content_type) + aioresponses.get(url, body=read_fixture_file(fixture), + content_type=content_type) @pytest.mark.usefixtures("populate_responses") @pytest.mark.asyncio From 8cec9bc9667daf9cf3e3eab74c3e447ca02c6368 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Wed, 29 May 2019 22:02:15 +0100 Subject: [PATCH 27/66] Make !hoogle async (untested) --- csbot/plugins/hoogle.py | 53 +++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/csbot/plugins/hoogle.py b/csbot/plugins/hoogle.py index e0ff7718..e42743b1 100644 --- a/csbot/plugins/hoogle.py +++ b/csbot/plugins/hoogle.py @@ -1,8 +1,7 @@ -import requests import urllib.parse from csbot.plugin import Plugin -from csbot.util import simple_http_get +from csbot.util import simple_http_get_async class Hoogle(Plugin): @@ -14,43 +13,45 @@ def setup(self): super(Hoogle, self).setup() @Plugin.command('hoogle') - def search_hoogle(self, e): + async def search_hoogle(self, e): """Search Hoogle with a given string and return the first few (exact number configurable) results. """ query = e['data'] hurl = 'http://www.haskell.org/hoogle/?mode=json&hoogle=' + query - hresp = simple_http_get(hurl) + async with simple_http_get_async(hurl) as hresp: - if hresp.status_code != requests.codes.ok: - self.log.warn('request failed for ' + hurl) - return + if hresp.status != 200: + self.log.warn('request failed for ' + hurl) + return + + # The Hoogle response JSON is of the following format: + # { + # "version": "" + # "results": [ + # { + # "location": "" + # "self": " :: " + # "docs": "" + # }, + # ... + # ] + # } + + maxresults = int(self.config_get('results')) + + json = await hresp.json() - # The Hoogle response JSON is of the following format: - # { - # "version": "" - # "results": [ - # { - # "location": "" - # "self": " :: " - # "docs": "" - # }, - # ... - # ] - # } - - maxresults = int(self.config_get('results')) - - if hresp.json is None: + if json is None: self.log.warn('invalid JSON received from Hoogle') return - if 'parseError' in hresp.json(): - e.reply(hresp.json()['parseError'].replace('\n', ' ')) + if 'parseError' in json: + e.reply(json['parseError'].replace('\n', ' ')) return - allresults = hresp.json()['results'] + allresults = json['results'] totalresults = len(allresults) results = allresults[0:maxresults] niceresults = [] From 73c1caf22ef4ea20031f1144d758fdb2c59029f3 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Wed, 29 May 2019 22:38:20 +0100 Subject: [PATCH 28/66] Make Imgur plugin async --- csbot/plugins/imgur.py | 56 ++++++++++++++++++++------------- csbot/test/test_plugin_imgur.py | 28 ++++++----------- csbot/util.py | 8 ++--- requirements.txt | 1 - 4 files changed, 47 insertions(+), 46 deletions(-) diff --git a/csbot/plugins/imgur.py b/csbot/plugins/imgur.py index 8f832d55..76e99a69 100644 --- a/csbot/plugins/imgur.py +++ b/csbot/plugins/imgur.py @@ -1,11 +1,12 @@ -from imgurpython import ImgurClient -from imgurpython.helpers.error import ImgurClientError - from ..plugin import Plugin -from ..util import pluralize +from ..util import pluralize, simple_http_get_async from .linkinfo import LinkInfoResult +class ImgurError(Exception): + pass + + class Imgur(Plugin): CONFIG_DEFAULTS = { 'client_id': None, @@ -17,17 +18,12 @@ class Imgur(Plugin): 'client_secret': ['IMGUR_CLIENT_SECRET'], } - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.client = ImgurClient(self.config_get('client_id'), - self.config_get('client_secret')) - @Plugin.integrate_with('linkinfo') def integrate_with_linkinfo(self, linkinfo): linkinfo.register_handler(lambda url: url.netloc in ('imgur.com', 'i.imgur.com'), self._linkinfo_handler, exclusive=True) - def _linkinfo_handler(self, url, match): + async def _linkinfo_handler(self, url, match): # Split up endpoint and ID: /, /a/ or /gallery/ kind, _, id = url.path.lstrip('/').rpartition('/') # Strip file extension from direct image links @@ -35,18 +31,18 @@ def _linkinfo_handler(self, url, match): try: if kind == '': - nsfw, title = self._format_image(self.client.get_image(id)) + nsfw, title = self._format_image(await self._get_image(id)) elif kind == 'a': - nsfw, title = self._format_album(self.client.get_album(id), url.fragment) + nsfw, title = self._format_album(await self._get_album(id), url.fragment) elif kind == 'gallery': - data = self.client.gallery_item(id) - if data.is_album: + data = await self._get_gallery_item(id) + if data['is_album']: nsfw, title = self._format_album(data, None) else: nsfw, title = self._format_image(data) else: nsfw, title = False, None - except ImgurClientError as e: + except ImgurError as e: return LinkInfoResult(url, str(e), is_error=True) if title: @@ -56,15 +52,33 @@ def _linkinfo_handler(self, url, match): @staticmethod def _format_image(data): - title = data.title or '' - return data.nsfw or 'nsfw' in title.lower(), title + title = data['title'] or '' + return data['nsfw'] or 'nsfw' in title.lower(), title @staticmethod def _format_album(data, image_id): - title = '{0} ({1})'.format(data.title or 'Untitled album', - pluralize(data.images_count, 'image', 'images')) - images = {i['id']: i for i in data.images} + title = '{0} ({1})'.format(data['title'] or 'Untitled album', + pluralize(data['images_count'], 'image', 'images')) + images = {i['id']: i for i in data['images']} image = images.get(image_id) if image and image['title']: title += ': ' + image['title'] - return data.nsfw or 'nsfw' in title.lower(), title + return data['nsfw'] or 'nsfw' in title.lower(), title + + async def _get(self, url): + headers = {'Authorization': f'Client-ID {self.config_get("client_id")}'} + async with simple_http_get_async(url, headers=headers) as resp: + json = await resp.json() + if json['success']: + return json['data'] + else: + raise ImgurError(json['data']['error']) + + async def _get_image(self, id): + return await self._get(f'https://api.imgur.com/3/image/{id}') + + async def _get_album(self, id): + return await self._get(f'https://api.imgur.com/3/album/{id}') + + async def _get_gallery_item(self, id): + return await self._get(f'https://api.imgur.com/3/gallery/{id}') diff --git a/csbot/test/test_plugin_imgur.py b/csbot/test/test_plugin_imgur.py index 0903497d..f92e47ea 100644 --- a/csbot/test/test_plugin_imgur.py +++ b/csbot/test/test_plugin_imgur.py @@ -164,20 +164,12 @@ """) -@pytest.fixture -def pre_irc_client(responses): - # imgurpython calls this on init to get initial rate limits - responses.add(responses.GET, 'https://api.imgur.com/3/credits', - status=200, body=read_fixture_file('imgur_credits.json'), - content_type='application/json') - - @pytest.mark.asyncio @pytest.mark.parametrize("url, api_url, status, content_type, fixture, title", test_cases) -async def test_integration(bot_helper, responses, url, api_url, status, content_type, fixture, title): - responses.add(responses.GET, api_url, status=status, - body=read_fixture_file(fixture), - content_type=content_type) +async def test_integration(bot_helper, aioresponses, url, api_url, status, content_type, fixture, title): + aioresponses.get(api_url, status=status, + body=read_fixture_file(fixture), + content_type=content_type) result = await bot_helper['linkinfo'].get_link_info(url) if title is None: assert result.is_error @@ -188,10 +180,10 @@ async def test_integration(bot_helper, responses, url, api_url, status, content_ @pytest.mark.asyncio @pytest.mark.parametrize("url, api_url, status, content_type, fixture, title", nsfw_test_cases) -async def test_integration_nsfw(bot_helper, responses, url, api_url, status, content_type, fixture, title): - responses.add(responses.GET, api_url, status=status, - body=read_fixture_file(fixture), - content_type=content_type) +async def test_integration_nsfw(bot_helper, aioresponses, url, api_url, status, content_type, fixture, title): + aioresponses.get(api_url, status=status, + body=read_fixture_file(fixture), + content_type=content_type) result = await bot_helper['linkinfo'].get_link_info(url) if title is None: assert result.is_error @@ -201,9 +193,7 @@ async def test_integration_nsfw(bot_helper, responses, url, api_url, status, con @pytest.mark.asyncio -async def test_invalid_URL(bot_helper, responses): +async def test_invalid_URL(bot_helper, aioresponses): """Test that an unrecognised URL never even results in a request.""" - responses.reset() # Drop requests used/made during plugin setup result = await bot_helper['linkinfo'].get_link_info('http://imgur.com/invalid/url') assert result.is_error - assert len(responses.calls) == 0 diff --git a/csbot/util.py b/csbot/util.py index 214b214e..abe5d0aa 100644 --- a/csbot/util.py +++ b/csbot/util.py @@ -99,17 +99,15 @@ def simple_http_get(url, stream=False): @asynccontextmanager -async def simple_http_get_async(url): +async def simple_http_get_async(url, **kwargs): session_kwargs = { 'headers': { 'User-Agent': 'csbot/0.1', }, } - get_kwargs = { - 'ssl': False, - } + kwargs.setdefault('ssl', False) async with aiohttp.ClientSession(**session_kwargs) as session: - async with session.get(url, **get_kwargs) as resp: + async with session.get(url, **kwargs) as resp: yield resp diff --git a/requirements.txt b/requirements.txt index 8f515f61..cb831892 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,6 @@ requests>=2.9.1,<3.0.0 lxml>=2.3.5 google-api-python-client>=1.4.1,<2.0.0 oauth2client>=3,<4 # google-api-python-client dropped dep in a minor release, generates ImportError warnings -imgurpython>=1.1.6,<2.0.0 isodate>=0.5.1 aiohttp>=3.5.1,<4.0 async_generator>=1.10,<2.0 From 6c0cf3351ad6e749bab140d373d81ba64f3bd479 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Thu, 30 May 2019 11:18:29 +0100 Subject: [PATCH 29/66] Make Youtube plugin async --- csbot/plugins/youtube.py | 52 ++++++--------- csbot/test/test_plugin_youtube.py | 105 ++++++++++++++++-------------- requirements.txt | 3 +- 3 files changed, 76 insertions(+), 84 deletions(-) diff --git a/csbot/plugins/youtube.py b/csbot/plugins/youtube.py index 2dfad3a9..ead5d495 100644 --- a/csbot/plugins/youtube.py +++ b/csbot/plugins/youtube.py @@ -1,8 +1,8 @@ import datetime import urllib.parse as urlparse -import apiclient import isodate +from aiogoogle import Aiogoogle, HTTPError from ..plugin import Plugin from .linkinfo import LinkInfoResult @@ -39,11 +39,9 @@ def __init__(self, http_error): self.http_error = http_error def __str__(self): - s = '%s: %s' % (self.http_error.resp.status, self.http_error._get_reason()) - if self.http_error.resp.status == 400: - return s + ' - invalid API key?' - else: - return s + s = '%s: %s' % (self.http_error.res.status_code, + self.http_error.res.json['error']['message']) + return s class Youtube(Plugin): @@ -61,39 +59,29 @@ class Youtube(Plugin): RESPONSE = '"{title}" [{duration}] (by {uploader} at {uploaded}) | Views: {views} [{likes}]' CMD_RESPONSE = RESPONSE + ' | {link}' - #: Hook for mocking HTTP responses to Google API client - http = None - client = None - - def setup(self): - super().setup() - self.client = apiclient.discovery.build( - 'youtube', 'v3', - developerKey=self.config_get('api_key'), - http=self.http) - - def get_video_json(self, id): - response = self.client.videos()\ - .list(id=id, hl='en', part='snippet,contentDetails,statistics')\ - .execute(http=self.http) - if len(response['items']) == 0: - return None - else: - return response['items'][0] + async def get_video_json(self, id): + async with Aiogoogle(api_key=self.config_get('api_key')) as aiogoogle: + youtube_v3 = await aiogoogle.discover('youtube', 'v3') + request = youtube_v3.videos.list(id=id, hl='en', part='snippet,contentDetails,statistics') + response = await aiogoogle.as_api_key(request) + if len(response['items']) == 0: + return None + else: + return response['items'][0] - def _yt(self, url): + async def _yt(self, url): """Builds a nicely formatted version of youtube's own internal JSON""" vid_id = get_yt_id(url) if not vid_id: return None try: - json = self.get_video_json(vid_id) + json = await self.get_video_json(vid_id) if json is None: return None except (KeyError, ValueError): return None - except apiclient.errors.HttpError as e: + except HTTPError as e: # Chain our own exception that gets a more sanitised error message raise YoutubeError(e) from e @@ -155,10 +143,10 @@ def _yt(self, url): def linkinfo_integrate(self, linkinfo): """Handle recognised youtube urls.""" - def page_handler(url, match): + async def page_handler(url, match): """Handles privmsg urls.""" try: - response = self._yt(url) + response = await self._yt(url) if response: return LinkInfoResult(url.geturl(), self.RESPONSE.format(**response)) else: @@ -171,11 +159,11 @@ def page_handler(url, match): @Plugin.command('youtube') @Plugin.command('yt') - def all_hail_our_google_overlords(self, e): + async def all_hail_our_google_overlords(self, e): """I for one, welcome our Google overlords.""" try: - response = self._yt(urlparse.urlparse(e["data"])) + response = await self._yt(urlparse.urlparse(e["data"])) if not response: e.reply("Invalid video ID") else: diff --git a/csbot/test/test_plugin_youtube.py b/csbot/test/test_plugin_youtube.py index c2178e87..de3ea59e 100644 --- a/csbot/test/test_plugin_youtube.py +++ b/csbot/test/test_plugin_youtube.py @@ -1,12 +1,11 @@ -from unittest.mock import patch from distutils.version import StrictVersion +import re import pytest import urllib.parse as urlparse -import apiclient -from csbot.test import fixture_file -from csbot.plugins.youtube import Youtube, YoutubeError +from csbot.test import read_fixture_file +from csbot.plugins.youtube import YoutubeError #: Tests are (number, url, content-type, status, fixture, expected) @@ -60,58 +59,62 @@ "youtube_flibble.json", None ), -] -# Non-success HttpMock results are broken in older versions of google-api-python-client -if StrictVersion(apiclient.__version__) > StrictVersion("1.4.1"): - json_test_cases += [ - # Invalid API key (400 Bad Request) - ( - "dQw4w9WgXcQ", - 400, - "youtube_invalid_key.json", - YoutubeError - ), - - # Valid API key, but Youtube Data API not enabled (403 Forbidden) - ( - "dQw4w9WgXcQ", - 403, - "youtube_access_not_configured.json", - YoutubeError - ), - ] + # Invalid API key (400 Bad Request) + ( + "dQw4w9WgXcQ", + 400, + "youtube_invalid_key.json", + YoutubeError + ), + + # Valid API key, but Youtube Data API not enabled (403 Forbidden) + ( + "dQw4w9WgXcQ", + 403, + "youtube_access_not_configured.json", + YoutubeError + ), +] @pytest.fixture -def pre_irc_client(): +def pre_irc_client(aioresponses): # Use fixture JSON for API client setup - http = apiclient.http.HttpMock( - fixture_file('google-discovery-youtube-v3.json'), - {'status': '200'}) - with patch.object(Youtube, 'http', wraps=http): - yield + aioresponses.get('https://www.googleapis.com/discovery/v1/apis/youtube/v3/rest', + status=200, content_type='application/json', + body=read_fixture_file('google-discovery-youtube-v3.json'), + repeat=True) @pytest.mark.bot(config="""\ [@bot] plugins = youtube + + [youtube] + api_key = abc """) class TestYoutubePlugin: + @pytest.mark.asyncio @pytest.mark.parametrize("vid_id, status, fixture, expected", json_test_cases) - def test_ids(self, bot_helper, vid_id, status, fixture, expected): - http = apiclient.http.HttpMock(fixture_file(fixture), {'status': status}) - with patch.object(bot_helper['youtube'], 'http', wraps=http): - if expected is YoutubeError: - with pytest.raises(YoutubeError): - bot_helper['youtube']._yt(urlparse.urlparse(vid_id)) - else: - assert bot_helper['youtube']._yt(urlparse.urlparse(vid_id)) == expected + async def test_ids(self, bot_helper, aioresponses, vid_id, status, fixture, expected): + pattern = re.compile(rf'https://www.googleapis.com/youtube/v3/videos\?.*\bid={vid_id}\b.*') + aioresponses.get(pattern, status=status, content_type='application/json', + body=read_fixture_file(fixture)) + + if expected is YoutubeError: + with pytest.raises(YoutubeError): + await bot_helper['youtube']._yt(urlparse.urlparse(vid_id)) + else: + assert await bot_helper['youtube']._yt(urlparse.urlparse(vid_id)) == expected @pytest.mark.bot(config="""\ [@bot] plugins = linkinfo youtube + + [youtube] + api_key = abc """) class TestYoutubeLinkInfoIntegration: @pytest.fixture @@ -134,15 +137,17 @@ def bot_helper(self, bot_helper): "http://www.youtube.com/watch?v={}&feature=youtube_gdata_player", "http://youtu.be/{}", ]) - async def test_integration(self, bot_helper, vid_id, status, fixture, response, url): - http = apiclient.http.HttpMock(fixture_file(fixture), {'status': status}) - with patch.object(bot_helper['youtube'], 'http', wraps=http): - url = url.format(vid_id) - result = await bot_helper['linkinfo'].get_link_info(url) - if response is None or response is YoutubeError: - assert result.is_error - else: - for key in response: - if key == "link": - continue - assert response[key] in result.text + async def test_integration(self, bot_helper, aioresponses, vid_id, status, fixture, response, url): + pattern = re.compile(rf'https://www.googleapis.com/youtube/v3/videos\?.*\bid={vid_id}\b.*') + aioresponses.get(pattern, status=status, content_type='application/json', + body=read_fixture_file(fixture)) + + url = url.format(vid_id) + result = await bot_helper['linkinfo'].get_link_info(url) + if response is None or response is YoutubeError: + assert result.is_error + else: + for key in response: + if key == "link": + continue + assert response[key] in result.text diff --git a/requirements.txt b/requirements.txt index cb831892..ab060564 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,8 +4,7 @@ straight.plugin==1.4.0-post-1 pymongo>=3.6.0 requests>=2.9.1,<3.0.0 lxml>=2.3.5 -google-api-python-client>=1.4.1,<2.0.0 -oauth2client>=3,<4 # google-api-python-client dropped dep in a minor release, generates ImportError warnings +aiogoogle==0.1.11 isodate>=0.5.1 aiohttp>=3.5.1,<4.0 async_generator>=1.10,<2.0 From e2cdb9a11fd9d6f9d62fbdcb52d292f9f321bf0b Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Thu, 30 May 2019 20:01:23 +0100 Subject: [PATCH 30/66] Use Python 3.6 compatible fork of aiogoogle PR to upstream project: https://github.com/omarryhan/aiogoogle/pull/7 --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ab060564..3a943efc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,8 @@ straight.plugin==1.4.0-post-1 pymongo>=3.6.0 requests>=2.9.1,<3.0.0 lxml>=2.3.5 -aiogoogle==0.1.11 +#aiogoogle==0.1.11 +git+https://github.com/alanbriolat/aiogoogle.git@py36#egg=aiogoogle isodate>=0.5.1 aiohttp>=3.5.1,<4.0 async_generator>=1.10,<2.0 From b43399f36cbf47ef88e9184513140658bd68190d Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Thu, 30 May 2019 21:04:44 +0100 Subject: [PATCH 31/66] Replace remainder of aresponses usage with fork of aioresponses PR to upstream project: https://github.com/pnuckowski/aioresponses/pull/121 --- csbot/test/test_plugin_linkinfo.py | 24 +++++++++++------------- requirements.txt | 4 ++-- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/csbot/test/test_plugin_linkinfo.py b/csbot/test/test_plugin_linkinfo.py index 0ed6c9f6..7c651d1c 100644 --- a/csbot/test/test_plugin_linkinfo.py +++ b/csbot/test/test_plugin_linkinfo.py @@ -6,6 +6,7 @@ import pytest import asynctest.mock import aiohttp +from aioresponses import CallbackResult from csbot.plugin import Plugin import csbot.core @@ -238,18 +239,16 @@ class Bot(csbot.core.Bot): pytestmark = pytest.mark.bot(cls=Bot, config=CONFIG) @pytest.mark.asyncio - async def test_non_blocking_privmsg(self, event_loop, bot_helper, aresponses): - # TODO: use aioresponses instead, once it supports async callbacks + async def test_non_blocking_privmsg(self, event_loop, bot_helper, aioresponses): bot_helper.reset_mock() event = asyncio.Event(loop=event_loop) - async def handler(request): + async def handler(url, **kwargs): await event.wait() - return aresponses.Response(status=200, - headers={'Content-Type': 'text/html'}, - body=b'foo') - aresponses.add('example.com', '/', 'get', handler) + return CallbackResult(status=200, content_type='text/html', + body=b'foo') + aioresponses.get('http://example.com/', callback=handler) futures = bot_helper.receive([ ':nick!user@host PRIVMSG #channel :a', @@ -272,19 +271,18 @@ async def handler(request): ]) @pytest.mark.asyncio - async def test_non_blocking_command(self, event_loop, bot_helper, aresponses): + async def test_non_blocking_command(self, event_loop, bot_helper, aioresponses): # TODO: use aioresponses instead, once it supports async callbacks bot_helper.reset_mock() event = asyncio.Event(loop=event_loop) - async def handler(request): + async def handler(url, **kwargs): await event.wait() - return aresponses.Response(status=200, - headers={'Content-Type': 'application/octet-stream'}, - body=b'foo') + return CallbackResult(status=200, content_type='application/octet-stream', + body=b'foo') - aresponses.add('example.com', '/', 'get', handler) + aioresponses.get('http://example.com/', callback=handler) futures = bot_helper.receive([ ':nick!user@host PRIVMSG #channel :a', diff --git a/requirements.txt b/requirements.txt index 3a943efc..2a013702 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,8 +15,8 @@ rollbar pytest==4.2.0 pytest-asyncio==0.10.0 pytest-aiohttp==0.3.0 -aresponses==1.1.1 -aioresponses==0.6.0 +#aioresponses==0.6.0 +git+https://github.com/alanbriolat/aioresponses.git@callback-coroutines#egg=aioresponses pytest-cov==2.6.1 asynctest==0.12.2 aiofastforward==0.0.17 From 736fea72649aa471ab1c7e00cc7b0af3318e6f7d Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Fri, 31 May 2019 11:18:14 +0100 Subject: [PATCH 32/66] Do Rollbar deploy notification on container start --- Dockerfile | 2 ++ docker-entrypoint.sh | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/Dockerfile b/Dockerfile index 81f59578..a84dea40 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,8 @@ WORKDIR /app # Update base OS RUN apt-get -y update && apt-get -y upgrade +# Install useful tools +RUN apt-get -y install git curl # Install Python 3(.4) RUN apt-get -y install python3 python3-dev python-virtualenv # Install dependencies for Python libs diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 7e739d69..289b4f79 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -1,3 +1,12 @@ #!/bin/bash + +if [[ ! -z "$ROLLBAR_ACCESS_TOKEN" ]] ; then + curl https://api.rollbar.com/api/1/deploy/ \ + -F access_token=$ROLLBAR_ACCESS_TOKEN \ + -F environment=${ROLLBAR_ENV:-development} \ + -F revision=`git rev-parse --verify HEAD` \ + -F local_username=`whoami` +fi + . /venv/bin/activate exec $@ From c37300409518f2a6a9344ea84446cd510fefc86e Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Sat, 1 Jun 2019 11:55:36 +0100 Subject: [PATCH 33/66] Don't force commands to be coroutines, but allow coroutine commands --- csbot/core.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/csbot/core.py b/csbot/core.py index 367449d9..78738cf1 100644 --- a/csbot/core.py +++ b/csbot/core.py @@ -161,8 +161,7 @@ def privmsg(self, event): self.post_event(command) @Plugin.hook('core.command') - @asyncio.coroutine - def fire_command(self, event): + async def fire_command(self, event): """Dispatch a command event to its callback. """ # Ignore unknown commands @@ -170,9 +169,11 @@ def fire_command(self, event): return f, _, _ = self.commands[event['command']] - if not asyncio.iscoroutinefunction(f): - f = asyncio.coroutine(f) - yield from f(event) + # Done like this instead of `if asyncio.iscoroutinefunction(...):` because `f` is a + # csbot.plugin.LazyMethod object + result = f(event) + if asyncio.iscoroutine(result): + await result @Plugin.command('help', help=('help [command]: show help for command, or ' 'show available commands')) From 7edb990b2e7213fa46baab97d2ae85a61211f6c6 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Sat, 1 Jun 2019 13:31:00 +0100 Subject: [PATCH 34/66] Unify behaviour for code that accepts both functions & coroutines --- csbot/core.py | 7 ++---- csbot/events.py | 17 +++++++------- csbot/plugins/linkinfo.py | 7 ++---- csbot/test/test_plugin_linkinfo.py | 1 - csbot/util.py | 37 ++++++++++++++++++++++++++++++ 5 files changed, 49 insertions(+), 20 deletions(-) diff --git a/csbot/core.py b/csbot/core.py index 78738cf1..1717d846 100644 --- a/csbot/core.py +++ b/csbot/core.py @@ -9,6 +9,7 @@ from csbot.plugin import build_plugin_dict, PluginManager import csbot.events as events from csbot.events import Event, CommandEvent +from csbot.util import maybe_future_result from .irc import IRCClient, IRCUser @@ -169,11 +170,7 @@ async def fire_command(self, event): return f, _, _ = self.commands[event['command']] - # Done like this instead of `if asyncio.iscoroutinefunction(...):` because `f` is a - # csbot.plugin.LazyMethod object - result = f(event) - if asyncio.iscoroutine(result): - await result + await maybe_future_result(f(event), log=self.log) @Plugin.command('help', help=('help [command]: show help for command, or ' 'show available commands')) diff --git a/csbot/events.py b/csbot/events.py index 4f4f8d06..568422a6 100644 --- a/csbot/events.py +++ b/csbot/events.py @@ -4,7 +4,7 @@ import asyncio import logging -from csbot.util import parse_arguments +from csbot.util import parse_arguments, maybe_future LOG = logging.getLogger('csbot.events') @@ -200,14 +200,13 @@ def _run_events(self): self._handle_exception(exception=e) continue # If the handler returned an awaitable (e.g. coroutine object), try to schedule it - if result is None: - continue - try: - future = asyncio.ensure_future(result, loop=self.loop) - except TypeError: - LOG.exception('non-awaitable result %r handling event %r', result, event) - continue - new_futures.add(future) + future = maybe_future( + result, + log=LOG, + loop=self.loop, + ) + if future: + new_futures.add(future) self.new_events.clear() if len(new_futures) > 0: LOG.debug('got %s new futures', len(new_futures)) diff --git a/csbot/plugins/linkinfo.py b/csbot/plugins/linkinfo.py index d8b948da..019b69c4 100644 --- a/csbot/plugins/linkinfo.py +++ b/csbot/plugins/linkinfo.py @@ -12,7 +12,7 @@ import lxml.html from ..plugin import Plugin -from ..util import Struct, simple_http_get_async +from ..util import Struct, simple_http_get_async, maybe_future_result LinkInfoHandler = namedtuple('LinkInfoHandler', ['filter', 'handler', 'exclusive']) @@ -186,10 +186,7 @@ async def get_link_info(self, original_url): for h in self.handlers: match = h.filter(url) if match: - if asyncio.iscoroutinefunction(h.handler): - result = await h.handler(url, match) - else: - result = h.handler(url, match) + result = await maybe_future_result(h.handler(url, match), log=self.log) if result is not None: # Useful result, return it return result diff --git a/csbot/test/test_plugin_linkinfo.py b/csbot/test/test_plugin_linkinfo.py index 7c651d1c..69299168 100644 --- a/csbot/test/test_plugin_linkinfo.py +++ b/csbot/test/test_plugin_linkinfo.py @@ -272,7 +272,6 @@ async def handler(url, **kwargs): @pytest.mark.asyncio async def test_non_blocking_command(self, event_loop, bot_helper, aioresponses): - # TODO: use aioresponses instead, once it supports async callbacks bot_helper.reset_mock() event = asyncio.Event(loop=event_loop) diff --git a/csbot/util.py b/csbot/util.py index abe5d0aa..0044c18a 100644 --- a/csbot/util.py +++ b/csbot/util.py @@ -1,12 +1,17 @@ import shlex from itertools import tee from collections import OrderedDict +import asyncio +import logging import requests from async_generator import asynccontextmanager import aiohttp +LOG = logging.getLogger(__name__) + + class User(object): def __init__(self, raw): self.raw = raw @@ -315,3 +320,35 @@ def __repr__(self): return '{}({})'.format(self.__class__.__name__, ', '.join('{}={!r}'.format(k, getattr(self, k)) for k in self._fields)) + + +def maybe_future(result, *, on_error=None, log=LOG, loop=None): + """Make *result* a future if possible, otherwise return None. + + If *result* is not None but also not awaitable, it is passed to *on_error* + if supplied, otherwise logged as a warning on *log*. + """ + if result is None: + return None + try: + future = asyncio.ensure_future(result, loop=loop) + except TypeError: + if on_error: + on_error(result) + else: + log.warning('maybe_future() ignoring non-awaitable result %r', result) + return None + return future + + +async def maybe_future_result(result, **kwargs): + """Get actual result from *result*. + + If *result* is awaitable, return the result of awaiting it, otherwise just + return *result*. + """ + future = maybe_future(result, **kwargs) + if future: + return await result + else: + return future From b1e7ae73e51e9adbb18b56db692805600170dfaf Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Sat, 1 Jun 2019 13:47:18 +0100 Subject: [PATCH 35/66] Docstrings for HybridEventRunner --- csbot/events.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/csbot/events.py b/csbot/events.py index 568422a6..32e63683 100644 --- a/csbot/events.py +++ b/csbot/events.py @@ -159,6 +159,23 @@ def _run(self): class HybridEventRunner: + """ + A hybrid synchronous/asynchronous event runner. + + *get_handlers* is called for each event passed to :meth:`post_event`, and + should return an iterable of callables to handle that event, each of which + will be called with the event object. + + Events are processed in the order they are received, with all handlers for + an event being called before the handlers for the next event. If a handler + returns an awaitable, it is added to a set of asynchronous tasks to wait on. + + The future returned by :meth:`post_event` completes only when all events + have been processed and all asynchronous tasks have completed. + + :param get_handlers: Get functions to call for an event + :param loop: asyncio event loop to use (default: use current loop) + """ def __init__(self, get_handlers, loop=None): self.get_handlers = get_handlers self.loop = loop @@ -176,6 +193,13 @@ def __exit__(self, exc_type, exc_value, traceback): self.future = None def post_event(self, event): + """Post *event* to be handled soon. + + *event* is added to the queue of events. + + Returns a future which resolves when the handlers of *event* (and all + events generated during those handlers) have completed. + """ self.events.append(event) LOG.debug('added event %s, pending=%s', event, len(self.events)) self.new_events.set() @@ -184,6 +208,8 @@ def post_event(self, event): return self.future def _run_events(self): + """Run event handlers, accumulating awaitables as futures. + """ new_futures = set() while len(self.events) > 0: LOG.debug('processing events (%s remaining)', len(self.events)) @@ -213,6 +239,11 @@ def _run_events(self): return new_futures async def _run(self): + """Run the event runner loop. + + Process events and await futures until all events and handlers have been + processed. + """ # Use self as context manager so an escaping exception doesn't break # the event runner instance permanently (i.e. we clean up the future) with self: From 3e02e99e4fb8698210d2aeb4bf8ca67bf26471f8 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Sat, 1 Jun 2019 22:17:07 +0100 Subject: [PATCH 36/66] Simplify docker build --- Dockerfile | 27 +++++---------------------- docker-entrypoint.sh | 3 +-- 2 files changed, 6 insertions(+), 24 deletions(-) diff --git a/Dockerfile b/Dockerfile index a84dea40..8b1110d5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,28 +1,11 @@ -FROM ubuntu:18.04 - -# From python:3.5 docker image, set locale -ENV LANG C.UTF-8 +FROM python:3.7 VOLUME /app WORKDIR /app +COPY csbot ./csbot +COPY csbot.*.cfg requirements.txt run_csbot.py docker-entrypoint.sh ./ -# Update base OS -RUN apt-get -y update && apt-get -y upgrade -# Install useful tools -RUN apt-get -y install git curl -# Install Python 3(.4) -RUN apt-get -y install python3 python3-dev python-virtualenv -# Install dependencies for Python libs -RUN apt-get -y install libxml2-dev libxslt1-dev zlib1g-dev - -# Copy needed files to build docker image -ADD requirements.txt docker-entrypoint.sh ./ - -# Create virtualenv -RUN virtualenv -p python3 /venv -# Populate virtualenv -RUN ./docker-entrypoint.sh pip install --upgrade pip -RUN ./docker-entrypoint.sh pip install -r requirements.txt +RUN pip install -r requirements.txt ENTRYPOINT ["./docker-entrypoint.sh"] -CMD ["./run_csbot.py", "csbot.cfg"] +CMD ["./csbot.cfg"] diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 289b4f79..a48bf0b8 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -8,5 +8,4 @@ if [[ ! -z "$ROLLBAR_ACCESS_TOKEN" ]] ; then -F local_username=`whoami` fi -. /venv/bin/activate -exec $@ +exec ./run_csbot.py $@ From 30ed66347d322d8e718d3d69d14eb661fe124ee6 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Sat, 1 Jun 2019 22:41:25 +0100 Subject: [PATCH 37/66] Bake git commit ID into container build --- .gitignore | 2 +- Dockerfile | 3 +++ docker-entrypoint.sh | 3 ++- hooks/build | 2 ++ 4 files changed, 8 insertions(+), 2 deletions(-) create mode 100755 hooks/build diff --git a/.gitignore b/.gitignore index bbc75b2d..56fdeb2a 100644 --- a/.gitignore +++ b/.gitignore @@ -27,7 +27,7 @@ cabal-dev *.egg *.egg-info dist -build +build/ eggs parts bin diff --git a/Dockerfile b/Dockerfile index 8b1110d5..aaa678c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,5 +7,8 @@ COPY csbot.*.cfg requirements.txt run_csbot.py docker-entrypoint.sh ./ RUN pip install -r requirements.txt +ARG SOURCE_COMMIT +ENV SOURCE_COMMIT $SOURCE_COMMIT + ENTRYPOINT ["./docker-entrypoint.sh"] CMD ["./csbot.cfg"] diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index a48bf0b8..20531073 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -4,8 +4,9 @@ if [[ ! -z "$ROLLBAR_ACCESS_TOKEN" ]] ; then curl https://api.rollbar.com/api/1/deploy/ \ -F access_token=$ROLLBAR_ACCESS_TOKEN \ -F environment=${ROLLBAR_ENV:-development} \ - -F revision=`git rev-parse --verify HEAD` \ + -F revision=$SOURCE_COMMIT \ -F local_username=`whoami` fi +env exec ./run_csbot.py $@ diff --git a/hooks/build b/hooks/build new file mode 100755 index 00000000..b89a853e --- /dev/null +++ b/hooks/build @@ -0,0 +1,2 @@ +#!/bin/bash +docker build --build-arg SOURCE_COMMIT=$SOURCE_COMMIT -f $DOCKERFILE_PATH -t $IMAGE_NAME . From 6b20a2c78cb3dab7f7d2375d6740d73260193d7e Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Sat, 1 Jun 2019 23:01:55 +0100 Subject: [PATCH 38/66] Update deployment docker-compose.yml to pull from docker hub and auto-update --- docker-compose.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1ba4756e..6f3e3037 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,20 +2,27 @@ version: "3" services: bot: - build: . + image: alanbriolat/csbot:latest links: - mongodb - volumes: - - .:/app env_file: - ./deploy.env environment: MONGODB_URI: mongodb://mongodb:27017/csbot - command: ./run_csbot.py csbot.deploy.cfg --rollbar + command: csbot.deploy.cfg ports: - "127.0.0.1:8180:80" + labels: + - com.centurylinklabs.watchtower.enable=true mongodb: image: mongo:4.0 volumes: - ./mongodb-data:/data/db + + watchtower: + image: containrrr/watchtower + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: --label-enable --cleanup --interval 120 + restart: always From ea9ce1adf7f09a417b6dde154240174b27e74c5a Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Sat, 1 Jun 2019 23:23:22 +0100 Subject: [PATCH 39/66] Control bot config location in docker-compose.yml with environment variables --- docker-compose.yml | 4 +++- docker-entrypoint.sh | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6f3e3037..03f74d4d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,13 +3,15 @@ version: "3" services: bot: image: alanbriolat/csbot:latest + volumes: + - ${CSBOT_CONFIG_LOCAL:-./csbot.cfg}:/app/csbot.cfg links: - mongodb env_file: - ./deploy.env environment: MONGODB_URI: mongodb://mongodb:27017/csbot - command: csbot.deploy.cfg + command: ${CSBOT_CONFIG:-csbot.cfg} ports: - "127.0.0.1:8180:80" labels: diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 20531073..1e5c4dd6 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -8,5 +8,4 @@ if [[ ! -z "$ROLLBAR_ACCESS_TOKEN" ]] ; then -F local_username=`whoami` fi -env exec ./run_csbot.py $@ From eee7c36511b7169154ea4b8d9e52cadbf32ecd2e Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Sun, 2 Jun 2019 09:17:17 +0100 Subject: [PATCH 40/66] Make Watchtower optional, update README --- .gitignore | 6 ++++++ README.rst | 18 ++++++++++++++++++ docker-compose.yml | 2 +- 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 56fdeb2a..ed6ba218 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,9 @@ htmlcov # Linters .ropeproject/ + +# Project-specific files +csbot.cfg +.env +deploy.env +mongodb-data/ \ No newline at end of file diff --git a/README.rst b/README.rst index d257d3e3..db6afff7 100644 --- a/README.rst +++ b/README.rst @@ -29,6 +29,22 @@ Docker containers (a MongoDB instance and the bot):: $ docker-compose up +This will use the `published image`_. To build locally:: + + $ docker build -t alanbriolat/csbot:latest . + +Environment variables to expose to the bot, e.g. for sensitive configuration +values, should be defined in ``deploy.env``. Environment variables used in +``docker-compose.yml`` should be defined in ``.env``: + +========================== ================== =========== +Variable Default Description +========================== ================== =========== +``CSBOT_CONFIG_LOCAL`` ``./csbot.cfg`` Path to config file in host filesystem to mount at ``/app/csbot.cfg`` +``CSBOT_CONFIG`` ``csbot.cfg`` Path to config file in container, relative to ``/app`` +``CSBOT_WATCHTOWER`` ``false`` Set to ``true`` to use Watchtower_ to auto-update when published container is updated +========================== ================== =========== + Backup MongoDB once services are running:: $ docker-compose exec -T mongodb mongodump --archive --gzip --quiet > foo.mongodump.gz @@ -73,3 +89,5 @@ We're also using Travis-CI for continuous integration and continuous deployment. .. _asyncio: https://docs.python.org/3/library/asyncio.html .. _lxml: http://lxml.de/ .. _Docker Compose: https://docs.docker.com/compose/ +.. _published image: https://hub.docker.com/r/alanbriolat/csbot +.. _Watchtower: https://containrrr.github.io/watchtower/ \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 03f74d4d..964c89dd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: ports: - "127.0.0.1:8180:80" labels: - - com.centurylinklabs.watchtower.enable=true + - com.centurylinklabs.watchtower.enable=${CSBOT_WATCHTOWER:-false} mongodb: image: mongo:4.0 From 9e1cc50d1698c63165c48a87d9426f4133ee0345 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Sun, 2 Jun 2019 09:41:13 +0100 Subject: [PATCH 41/66] Enable automated testing on docker hub --- docker-compose.test.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 docker-compose.test.yml diff --git a/docker-compose.test.yml b/docker-compose.test.yml new file mode 100644 index 00000000..bfdaa413 --- /dev/null +++ b/docker-compose.test.yml @@ -0,0 +1,7 @@ +version: "3" + +services: + sut: + build: . + entrypoint: pytest + command: [] \ No newline at end of file From 46866476c27545154bc9dc872c618d394765570e Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Sun, 2 Jun 2019 10:00:49 +0100 Subject: [PATCH 42/66] Avoid some problems with local docker builds by removing .pyc files --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index aaa678c5..393ed32d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,7 @@ VOLUME /app WORKDIR /app COPY csbot ./csbot COPY csbot.*.cfg requirements.txt run_csbot.py docker-entrypoint.sh ./ +RUN find . -name '*.pyc' -delete RUN pip install -r requirements.txt From 39d4988159c03812e675e529c28242173f86bee0 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Sun, 2 Jun 2019 10:23:59 +0100 Subject: [PATCH 43/66] Remove heroku deployment from travis config --- .travis.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 03cde24e..b10f3e1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,12 +24,3 @@ notifications: - irc.freenode.org#cs-york-dev skip_join: true use_notice: true - -deploy: - provider: heroku - api_key: - secure: jHzS/L/cN/6gCNJrmVCVDb0V4+Zc1b/PnTYcVfoaAw7/USIb2ZQbU6uwPCpGZ8EL/dQlgOCwJY1UYzowm5d6xvXw+9+iHOIBAAgPe0VEmJ2GMPd1/n8cl5CiJ+LF3NXyBml/F4BL/2wm+kZUxINeZfJaim2OAd9g8PfgpHUey5A= - app: csyorkbot - on: - repo: HackSoc/csbot - python: '3.6' From e660c3dcdcc07b2a4ec4121160a1df03d7400aed Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Sun, 2 Jun 2019 18:04:44 +0100 Subject: [PATCH 44/66] Move rollbar deploy notification to bot startup code --- csbot/__init__.py | 41 ++++++++++++++++++++++++++++++++++------- docker-entrypoint.sh | 8 -------- 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/csbot/__init__.py b/csbot/__init__.py index 0e92fbd5..420142cc 100644 --- a/csbot/__init__.py +++ b/csbot/__init__.py @@ -4,11 +4,15 @@ import os import click +import aiohttp import rollbar from .core import Bot +LOG = logging.getLogger(__name__) + + @click.command(context_settings={'help_option_names': ['-h', '--help']}) @click.option('--debug', '-d', is_flag=True, default=False, help='Turn on debug logging for the bot.') @@ -22,10 +26,12 @@ help='Turn on all debug logging.') @click.option('--colour/--no-colour', 'colour_logging', default=None, help='Use colour in logging. [default: automatic]') -@click.option('--rollbar/--no-rollbar', 'use_rollbar', default=False, - help='Enable Rollbar error reporting.') +@click.option('--rollbar-token', default=None, + help='Rollbar access token, enables Rollbar error reporting.') +@click.option('--env-name', default='development', + help='Deployment environment name. [default: development]') @click.argument('config', type=click.File('r')) -def main(config, debug, debug_irc, debug_events, debug_asyncio, debug_all, colour_logging, use_rollbar): +def main(config, debug, debug_irc, debug_events, debug_asyncio, debug_all, colour_logging, rollbar_token, env_name): """Run an IRC bot from a configuration file. """ # Apply "debug all" option @@ -73,10 +79,9 @@ def main(config, debug, debug_irc, debug_events, debug_asyncio, debug_all, colou client = Bot(config) client.bot_setup() - # Configure Rollbar for exception reporting - if use_rollbar: - rollbar.init(os.environ['ROLLBAR_ACCESS_TOKEN'], - os.environ.get('ROLLBAR_ENV', 'development')) + # Configure Rollbar for exception reporting, report deployment + if rollbar_token: + rollbar.init(rollbar_token, env_name) def handler(loop, context): exception = context.get('exception') @@ -88,6 +93,28 @@ def handler(loop, context): loop.default_exception_handler(context) client.loop.set_exception_handler(handler) + async def rollbar_report_deploy(revision): + async with aiohttp.ClientSession() as session: + request = session.post( + 'https://api.rollbar.com/api/1/deploy/', + data={ + 'access_token': rollbar_token, + 'environment': env_name, + 'revision': revision, + }, + ) + async with request as response: + data = await response.json() + if response.status == 200: + LOG.info('Reported deploy to Rollbar: env=%s revision=%s deploy_id=%s', + env_name, revision, data['data']['deploy_id']) + else: + LOG.error('Error reporting deploy to Rollbar: %s', data['message']) + + revision = os.environ.get('SOURCE_COMMIT', None) + if revision: + client.loop.run_until_complete(rollbar_report_deploy(revision)) + # Run the client def stop(): client.disconnect() diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 1e5c4dd6..ffec0da1 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -1,11 +1,3 @@ #!/bin/bash -if [[ ! -z "$ROLLBAR_ACCESS_TOKEN" ]] ; then - curl https://api.rollbar.com/api/1/deploy/ \ - -F access_token=$ROLLBAR_ACCESS_TOKEN \ - -F environment=${ROLLBAR_ENV:-development} \ - -F revision=$SOURCE_COMMIT \ - -F local_username=`whoami` -fi - exec ./run_csbot.py $@ From 94131e15170457051e16e19e08947e1b57b4e017 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Sun, 2 Jun 2019 22:47:50 +0100 Subject: [PATCH 45/66] Add GitHub deploy notification --- csbot/__init__.py | 103 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 82 insertions(+), 21 deletions(-) diff --git a/csbot/__init__.py b/csbot/__init__.py index 420142cc..c228c650 100644 --- a/csbot/__init__.py +++ b/csbot/__init__.py @@ -28,12 +28,28 @@ help='Use colour in logging. [default: automatic]') @click.option('--rollbar-token', default=None, help='Rollbar access token, enables Rollbar error reporting.') +@click.option('--github-token', default=None, + help='GitHub "personal access token", enables GitHub deployment reporting.') +@click.option('--github-repo', default=None, + help='GitHub repository to report deployments to.') @click.option('--env-name', default='development', help='Deployment environment name. [default: development]') @click.argument('config', type=click.File('r')) -def main(config, debug, debug_irc, debug_events, debug_asyncio, debug_all, colour_logging, rollbar_token, env_name): +def main(config, + debug, + debug_irc, + debug_events, + debug_asyncio, + debug_all, + colour_logging, + rollbar_token, + github_token, + github_repo, + env_name): """Run an IRC bot from a configuration file. """ + revision = os.environ.get('SOURCE_COMMIT', None) + # Apply "debug all" option if debug_all: debug = debug_irc = debug_events = debug_asyncio = True @@ -93,27 +109,11 @@ def handler(loop, context): loop.default_exception_handler(context) client.loop.set_exception_handler(handler) - async def rollbar_report_deploy(revision): - async with aiohttp.ClientSession() as session: - request = session.post( - 'https://api.rollbar.com/api/1/deploy/', - data={ - 'access_token': rollbar_token, - 'environment': env_name, - 'revision': revision, - }, - ) - async with request as response: - data = await response.json() - if response.status == 200: - LOG.info('Reported deploy to Rollbar: env=%s revision=%s deploy_id=%s', - env_name, revision, data['data']['deploy_id']) - else: - LOG.error('Error reporting deploy to Rollbar: %s', data['message']) - - revision = os.environ.get('SOURCE_COMMIT', None) if revision: - client.loop.run_until_complete(rollbar_report_deploy(revision)) + client.loop.run_until_complete(rollbar_report_deploy(rollbar_token, env_name, revision)) + + if github_token and github_repo and revision: + client.loop.run_until_complete(github_report_deploy(github_token, github_repo, env_name, revision)) # Run the client def stop(): @@ -127,6 +127,67 @@ def stop(): client.bot_teardown() +async def rollbar_report_deploy(rollbar_token, env_name, revision): + async with aiohttp.ClientSession() as session: + request = session.post( + 'https://api.rollbar.com/api/1/deploy/', + data={ + 'access_token': rollbar_token, + 'environment': env_name, + 'revision': revision, + }, + ) + async with request as response: + data = await response.json() + if response.status == 200: + LOG.info('Reported deploy to Rollbar: env=%s revision=%s deploy_id=%s', + env_name, revision, data['data']['deploy_id']) + else: + LOG.error('Error reporting deploy to Rollbar: %s', data['message']) + + +async def github_report_deploy(github_token, github_repo, env_name, revision): + headers = { + 'Authorization': f'token {github_token}', + 'Accept': 'application/vnd.github.v3+json', + } + async with aiohttp.ClientSession(headers=headers) as session: + create_request = session.post( + f'https://api.github.com/repos/{github_repo}/deployments', + json={ + 'ref': revision, + 'auto_merge': False, + 'environment': env_name, + 'description': 'Bot running with new version', + }, + ) + async with create_request as create_response: + if create_response.status != 201: + LOG.error('Error reporting deploy to GitHub (create deploy): %s %s\n%s', + create_response.status, create_response.reason, await create_response.text()) + return + + deploy = await create_response.json() + + status_request = session.post( + deploy['statuses_url'], + json={ + 'state': 'success', + + }, + ) + async with status_request as status_response: + if status_response.status != 201: + LOG.error('Error reporting deploy to GitHub (update status): %s %s\n%s', + create_response.status, create_response.reason, await create_response.text()) + return + + status = await status_response.json() + + LOG.info('Reported deploy to GitHub: env=%s revision=%s deploy_id=%s', + env_name, revision, deploy["id"]) + + class PrettyStreamHandler(logging.StreamHandler): """Wrap log messages with severity-dependent ANSI terminal colours. From f4194b2f38719725645002ee0ef7f64238d547ab Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Mon, 3 Jun 2019 12:30:51 +0100 Subject: [PATCH 46/66] Upstream PR to aiogoogle now released --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2a013702..2ed71c53 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,8 +4,7 @@ straight.plugin==1.4.0-post-1 pymongo>=3.6.0 requests>=2.9.1,<3.0.0 lxml>=2.3.5 -#aiogoogle==0.1.11 -git+https://github.com/alanbriolat/aiogoogle.git@py36#egg=aiogoogle +aiogoogle==0.1.13 isodate>=0.5.1 aiohttp>=3.5.1,<4.0 async_generator>=1.10,<2.0 From de5b687d334c72a1a729a5ef014350bdb444527d Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Mon, 3 Jun 2019 16:39:33 +0100 Subject: [PATCH 47/66] Fix some lint --- csbot/core.py | 1 - csbot/plugin.py | 1 - csbot/plugins/linkinfo.py | 1 - csbot/test/test_events.py | 4 ++-- csbot/test/test_plugin_github.py | 6 +++--- csbot/test/test_plugin_linkinfo.py | 2 +- csbot/test/test_plugin_youtube.py | 5 ++--- 7 files changed, 8 insertions(+), 12 deletions(-) diff --git a/csbot/core.py b/csbot/core.py index 1717d846..bda920d7 100644 --- a/csbot/core.py +++ b/csbot/core.py @@ -1,6 +1,5 @@ import collections import itertools -import asyncio import configparser import straight.plugin diff --git a/csbot/plugin.py b/csbot/plugin.py index 61124491..e52fa63f 100644 --- a/csbot/plugin.py +++ b/csbot/plugin.py @@ -2,7 +2,6 @@ from collections import abc import logging import os -import asyncio from typing import List, Callable diff --git a/csbot/plugins/linkinfo.py b/csbot/plugins/linkinfo.py index 019b69c4..7dbca490 100644 --- a/csbot/plugins/linkinfo.py +++ b/csbot/plugins/linkinfo.py @@ -5,7 +5,6 @@ from collections import namedtuple import datetime from functools import partial -import asyncio import aiohttp import lxml.etree diff --git a/csbot/test/test_events.py b/csbot/test/test_events.py index eeb38749..cb09f455 100644 --- a/csbot/test/test_events.py +++ b/csbot/test/test_events.py @@ -339,7 +339,7 @@ async def e(_): mock.call('a'), ] assert complete == ['a1'] - + # Unblock a2 and allow some tasks to run: # - a2 should complete # - post_event('b') should be called (by a2) @@ -359,7 +359,7 @@ async def e(_): mock.call('f'), ] assert complete == ['a1', 'a2', 'b1', 'b2', 'c', 'd'] - + # Unblock b3 and allow some tasks to run: # - b3 should complete # - post_event('e') should be called (by b3) diff --git a/csbot/test/test_plugin_github.py b/csbot/test/test_plugin_github.py index 8efaec69..490d625e 100644 --- a/csbot/test/test_plugin_github.py +++ b/csbot/test/test_plugin_github.py @@ -54,7 +54,7 @@ class TestGitHubPlugin: fmt.issue_text = {issue[title]} ({issue[html_url]}) fmt.pr_num = PR #{pull_request[number]} fmt.pr_text = {pull_request[title]} ({pull_request[html_url]}) - + # Format strings for specific events fmt/create = {fmt.source} created {ref_type} {ref} ({repository[html_url]}/tree/{ref}) fmt/delete = {fmt.source} deleted {ref_type} {ref} @@ -67,7 +67,7 @@ class TestGitHubPlugin: fmt/push/pushed = {fmt.source} pushed {count} new commit(s) to {short_ref}: {compare} fmt/push/forced = {fmt.source} updated {short_ref}: {compare} fmt/release/* = {fmt.source} {event_subtype} release {release[name]}: {release[html_url]} - + [github/alanbriolat/csbot-webhook-test] notify = #mychannel """ @@ -217,7 +217,7 @@ async def test_behaviour(self, bot_helper, client, fixture_file, expected): [webhook] url_secret = test_url [github] -secret = +secret = """) async def test_signature_ignored(bot_helper, client): """X-Hub-Signature invalid, but secret is blank, so not verified and handler called""" diff --git a/csbot/test/test_plugin_linkinfo.py b/csbot/test/test_plugin_linkinfo.py index 69299168..38560568 100644 --- a/csbot/test/test_plugin_linkinfo.py +++ b/csbot/test/test_plugin_linkinfo.py @@ -165,7 +165,7 @@ async def test_not_found(bot_helper, aioresponses): # Test our assumptions: direct request should raise connection error, because aioresponses # is mocking the internet with pytest.raises(aiohttp.ClientConnectionError): - async with aiohttp.ClientSession() as session, session.get('http://example.com/') as resp: + async with aiohttp.ClientSession() as session, session.get('http://example.com/'): pass # Should result in an error message from linkinfo (and implicitly no exception raised) diff --git a/csbot/test/test_plugin_youtube.py b/csbot/test/test_plugin_youtube.py index de3ea59e..8469fdd5 100644 --- a/csbot/test/test_plugin_youtube.py +++ b/csbot/test/test_plugin_youtube.py @@ -1,4 +1,3 @@ -from distutils.version import StrictVersion import re import pytest @@ -90,7 +89,7 @@ def pre_irc_client(aioresponses): @pytest.mark.bot(config="""\ [@bot] plugins = youtube - + [youtube] api_key = abc """) @@ -112,7 +111,7 @@ async def test_ids(self, bot_helper, aioresponses, vid_id, status, fixture, expe @pytest.mark.bot(config="""\ [@bot] plugins = linkinfo youtube - + [youtube] api_key = abc """) From dde4d5b0cb3ee89ec506cc21dc5d71287dcc2869 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Mon, 3 Jun 2019 16:57:30 +0100 Subject: [PATCH 48/66] Remove commented broken feature --- csbot/irc.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/csbot/irc.py b/csbot/irc.py index 45d5d335..be402292 100644 --- a/csbot/irc.py +++ b/csbot/irc.py @@ -373,9 +373,6 @@ async def connection_made(self): if self.__config['ircv3']: self.send(IRCMessage.create('CAP', ['END'])) - # TODO: uncomment this? tests hang if uncommented... - # await self.wait_for_message(lambda m: (m.command_name == 'RPL_WELCOME', m)) - self._start_client_pings() async def connection_lost(self, exc): From 2cbc25ff3919ee459acc89ba1c5e5d435f120aa9 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Mon, 3 Jun 2019 22:21:22 +0100 Subject: [PATCH 49/66] Enable IRCv3 option for deployment --- csbot.deploy.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/csbot.deploy.cfg b/csbot.deploy.cfg index 0ae6e913..10207210 100644 --- a/csbot.deploy.cfg +++ b/csbot.deploy.cfg @@ -1,4 +1,5 @@ [@bot] +ircv3 = true nickname = Mathison auth_method = sasl_plain channels = #cs-york #cs-york-dev #compsoc-uk #hacksoc #hacksoc-bottest From d9f5bee47b2b2d068c4865020dfd22fbb6a98e3e Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Tue, 4 Jun 2019 11:00:58 +0100 Subject: [PATCH 50/66] Test and fix maybe_future/maybe_future_result Fixes #158 --- csbot/test/test_util.py | 60 +++++++++++++++++++++++++++++++++++++++++ csbot/util.py | 4 +-- 2 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 csbot/test/test_util.py diff --git a/csbot/test/test_util.py b/csbot/test/test_util.py new file mode 100644 index 00000000..3413fa28 --- /dev/null +++ b/csbot/test/test_util.py @@ -0,0 +1,60 @@ +import asyncio +from unittest import mock + +import pytest + +from csbot import util + + +@pytest.mark.asyncio +async def test_maybe_future_none(): + assert util.maybe_future(None) is None + + +@pytest.mark.asyncio +async def test_maybe_future_non_awaitable(): + on_error = mock.Mock(spec=callable) + assert util.maybe_future("foo", on_error=on_error) is None + assert on_error.mock_calls == [ + mock.call("foo"), + ] + + +@pytest.mark.asyncio +async def test_maybe_future_coroutine(): + async def foo(): + await asyncio.sleep(0) + return "bar" + + future = util.maybe_future(foo()) + assert future is not None + assert not future.done() + await future + assert future.done() + assert future.exception() is None + + +@pytest.mark.asyncio +async def test_maybe_future_result_none(): + result = await util.maybe_future_result(None) + assert result is None + + +@pytest.mark.asyncio +async def test_maybe_future_result_non_awaitable(): + on_error = mock.Mock(spec=callable) + result = await util.maybe_future_result("foo", on_error=on_error) + assert result == "foo" + assert on_error.mock_calls == [ + mock.call("foo"), + ] + + +@pytest.mark.asyncio +async def test_maybe_future_result_coroutine(): + async def foo(): + await asyncio.sleep(0) + return "bar" + + result = await util.maybe_future_result(foo()) + assert result == "bar" diff --git a/csbot/util.py b/csbot/util.py index 0044c18a..3e39de11 100644 --- a/csbot/util.py +++ b/csbot/util.py @@ -349,6 +349,6 @@ async def maybe_future_result(result, **kwargs): """ future = maybe_future(result, **kwargs) if future: - return await result + return await future else: - return future + return result From 8491eafc14f740cc1faa68df9f4a98edb9509f1f Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Thu, 6 Jun 2019 16:49:18 +0100 Subject: [PATCH 51/66] Include current event and recent messages in rollbar reports --- csbot/__init__.py | 6 ++++- csbot/core.py | 7 ++++++ csbot/events.py | 49 +++++++++++++++++++++++++++------------ csbot/test/test_events.py | 5 ++++ 4 files changed, 51 insertions(+), 16 deletions(-) diff --git a/csbot/__init__.py b/csbot/__init__.py index c228c650..f04522c6 100644 --- a/csbot/__init__.py +++ b/csbot/__init__.py @@ -105,7 +105,11 @@ def handler(loop, context): exc_info = (type(exception), exception, exception.__traceback__) else: exc_info = None - rollbar.report_exc_info(exc_info) + extra_data = { + 'csbot_event': context.get('csbot_event'), + 'csbot_recent_messages': "\n".join(client.recent_messages), + } + rollbar.report_exc_info(exc_info, extra_data=extra_data) loop.default_exception_handler(context) client.loop.set_exception_handler(handler) diff --git a/csbot/core.py b/csbot/core.py index bda920d7..7bab9b15 100644 --- a/csbot/core.py +++ b/csbot/core.py @@ -83,6 +83,8 @@ def __init__(self, config=None, loop=None): client_ping_interval=int(self.config_get('client_ping')), ) + self._recent_messages = collections.deque(maxlen=10) + # Plumb in reply(...) method if self.config_getboolean('use_notice'): self.reply = self.notice @@ -217,10 +219,15 @@ def send_line(self, line): self.emit_new('core.raw.sent', {'message': line}) def line_received(self, line): + self._recent_messages.append(line) fut = self.emit_new('core.raw.received', {'message': line}) super().line_received(line) return fut + @property + def recent_messages(self): + return list(self._recent_messages) + def on_welcome(self): self.emit_new('core.self.connected') diff --git a/csbot/events.py b/csbot/events.py index 32e63683..7309c6a8 100644 --- a/csbot/events.py +++ b/csbot/events.py @@ -220,17 +220,7 @@ def _run_events(self): for handler in self.get_handlers(event): # Attempt to run the handler, but don't break everything if the handler fails LOG.debug('running handler: %r', handler) - try: - result = handler(event) - except Exception as e: - self._handle_exception(exception=e) - continue - # If the handler returned an awaitable (e.g. coroutine object), try to schedule it - future = maybe_future( - result, - log=LOG, - loop=self.loop, - ) + future = self._run_handler(handler, event) if future: new_futures.add(future) self.new_events.clear() @@ -238,6 +228,34 @@ def _run_events(self): LOG.debug('got %s new futures', len(new_futures)) return new_futures + def _run_handler(self, handler, event): + """Call *handler* with *event* and log any exception. + + If *handler* returns an awaitable, then it is wrapped in a coroutine that will log any + exception from awaiting it. + """ + result = None + try: + result = handler(event) + except Exception as e: + self._handle_exception(exception=e, csbot_event=event) + future = maybe_future( + result, + log=LOG, + loop=self.loop, + ) + if future: + future = asyncio.ensure_future(self._finish_async_handler(future, event), loop=self.loop) + return future + + async def _finish_async_handler(self, future, event): + """Await *future* and log any exception. + """ + try: + await future + except Exception as e: + self._handle_exception(future=future, csbot_event=event) + async def _run(self): """Run the event runner loop. @@ -266,22 +284,23 @@ async def _run(self): done_futures = done - {new_events} LOG.debug('%s of %s futures done', len(done_futures), len(self.futures)) self.futures -= done_futures - for f in done_futures: - if f.exception() is not None: - self._handle_exception(future=f) if new_events.done(): LOG.debug('new events to process') else: # If no new events, cancel the waiter, because we'll create a new one next iteration new_events.cancel() - def _handle_exception(self, *, message='Unhandled exception in event handler', exception=None, future=None): + def _handle_exception(self, *, message='Unhandled exception in event handler', + exception=None, + future=None, + csbot_event=None): if exception is None and future is not None: exception = future.exception() self.loop.call_exception_handler({ 'message': message, 'exception': exception, 'future': future, + 'csbot_event': csbot_event, }) diff --git a/csbot/test/test_events.py b/csbot/test/test_events.py index cb09f455..750d8fe0 100644 --- a/csbot/test/test_events.py +++ b/csbot/test/test_events.py @@ -581,6 +581,11 @@ async def b2(_): assert complete == ['a2', 'a4', 'b2'] assert event_runner.exception_handler.call_count == 3 + # Check that exception handler calls have the correct event context + assert event_runner.exception_handler.mock_calls[0][1][1]['csbot_event'] == 'a' + assert event_runner.exception_handler.mock_calls[1][1][1]['csbot_event'] == 'a' + assert event_runner.exception_handler.mock_calls[2][1][1]['csbot_event'] == 'b' + class TestEvent(unittest.TestCase): class DummyBot(object): From b00f4ef518c35b0bb86c98f85c1f1d5732ac4378 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Thu, 6 Jun 2019 22:20:49 +0100 Subject: [PATCH 52/66] Only send client PING during silence from the server --- csbot/irc.py | 21 ++++++++++++++++++--- csbot/test/test_irc.py | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/csbot/irc.py b/csbot/irc.py index be402292..48ff86ae 100644 --- a/csbot/irc.py +++ b/csbot/irc.py @@ -262,6 +262,7 @@ def __init__(self, *, loop=None, **kwargs): self.connected.clear() self.disconnected = asyncio.Event(loop=self.loop) self.disconnected.set() + self._last_message_received = self.loop.time() self._client_ping = None self._client_ping_counter = 0 @@ -387,6 +388,7 @@ async def connection_lost(self, exc): def line_received(self, line): """Callback for received raw IRC message.""" + self._last_message_received = self.loop.time() msg = IRCMessage.parse(line) LOG.debug('>>> %s', msg.pretty) self.message_received(msg) @@ -424,11 +426,24 @@ def _stop_client_pings(self): self._client_ping = None async def _send_client_pings(self, interval): + """Send a client ``PING`` if no messages have been received for *interval* seconds.""" self._client_ping_counter = 0 + delay = interval while True: - await asyncio.sleep(interval) - self._client_ping_counter += 1 - self.send_line(f'PING {self._client_ping_counter}') + await asyncio.sleep(delay) + now = self.loop.time() + remaining = self._last_message_received + interval - now + + if remaining <= 0: + # Send the PING + self._client_ping_counter += 1 + self.send_line(f'PING {self._client_ping_counter}') + # Wait for another interval + delay = interval + else: + # Wait until interval has elapsed since last message + delay = remaining + class Waiter: predicate = None diff --git a/csbot/test/test_irc.py b/csbot/test/test_irc.py index 46c04e00..2a652885 100644 --- a/csbot/test/test_irc.py +++ b/csbot/test/test_irc.py @@ -132,8 +132,9 @@ def irc_client_config(self): 'client_ping_interval': 3, } - @pytest.mark.asyncio(foo='bar') + @pytest.mark.asyncio async def test_client_PING(self, fast_forward, run_client): + """Check that client PING commands are sent at the expected interval.""" run_client.reset_mock() run_client.client.send_line.assert_not_called() # Advance time, test that a ping was sent @@ -161,6 +162,39 @@ async def test_client_PING(self, fast_forward, run_client): mock.call('PING 5'), ] + @pytest.mark.asyncio + async def test_client_PING_only_when_needed(self, fast_forward, run_client): + """Check that client PING commands are sent relative to the last received message.""" + run_client.reset_mock() + run_client.client.send_line.assert_not_called() + # Advance time to just before the second PING, check that the first PING was sent + await fast_forward(5) + assert run_client.client.send_line.mock_calls == [ + mock.call('PING 1'), + ] + # Receive a message, this should reset the PING timer + run_client.receive(':nick!user@host PRIVMSG #channel :foo') + # Advance time to just after when the second PING would happen without any messages + # received, check that still only one PING was sent + await fast_forward(2) + assert run_client.client.send_line.mock_calls == [ + mock.call('PING 1'), + ] + # Advance time to 4 seconds after the last message was received, and check that another + # PING has now been sent + await fast_forward(2) + assert run_client.client.send_line.mock_calls == [ + mock.call('PING 1'), + mock.call('PING 2'), + ] + # Disconnect, advance time, test that no more pings were sent + run_client.client.disconnect() + await fast_forward(12) + assert run_client.client.send_line.mock_calls == [ + mock.call('PING 1'), + mock.call('PING 2'), + ] + def test_PING_PONG(irc_client_helper): irc_client_helper.receive('PING :i.am.a.server') From 5ff53856bf206491698b541e864a2e2effcd6665 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Fri, 7 Jun 2019 12:02:08 +0100 Subject: [PATCH 53/66] Refactor project layout: source code in src/, tests in tests/ --- setup.cfg | 2 ++ setup.py | 12 ++++++++---- {csbot => src/csbot}/__init__.py | 0 {csbot => src/csbot}/_rfc.py | 0 {csbot => src/csbot}/core.py | 0 {csbot => src/csbot}/events.py | 0 {csbot => src/csbot}/irc.py | 0 {csbot => src/csbot}/plugin.py | 0 {csbot => src/csbot}/plugins/__init__.py | 0 {csbot => src/csbot}/plugins/auth.py | 0 {csbot => src/csbot}/plugins/calc.py | 0 {csbot => src/csbot}/plugins/cron.py | 0 {csbot => src/csbot}/plugins/csyork.py | 0 {csbot => src/csbot}/plugins/github.py | 0 {csbot => src/csbot}/plugins/helix.py | 0 {csbot => src/csbot}/plugins/hoogle.py | 0 {csbot => src/csbot}/plugins/imgur.py | 0 {csbot => src/csbot}/plugins/last.py | 0 {csbot => src/csbot}/plugins/linkinfo.py | 0 {csbot => src/csbot}/plugins/logger.py | 0 {csbot => src/csbot}/plugins/mongodb.py | 0 {csbot => src/csbot}/plugins/termdates.py | 0 {csbot => src/csbot}/plugins/topic.py | 0 {csbot => src/csbot}/plugins/usertrack.py | 0 {csbot => src/csbot}/plugins/webhook.py | 0 {csbot => src/csbot}/plugins/webserver.py | 0 {csbot => src/csbot}/plugins/whois.py | 0 {csbot => src/csbot}/plugins/xkcd.py | 0 {csbot => src/csbot}/plugins/youtube.py | 0 {csbot => src/csbot}/plugins_broken/tell.py | 0 {csbot => src/csbot}/plugins_broken/users.py | 0 {csbot => src/csbot}/util.py | 0 {csbot/test => tests}/__init__.py | 0 {csbot/test => tests}/conftest.py | 6 +++--- {csbot/test => tests}/fixtures/empty_file | 0 .../github-create-20190129-215300.headers.json | 0 .../github-create-20190129-215300.payload.json | 0 .../github-create-20190130-101054.headers.json | 0 .../github-create-20190130-101054.payload.json | 0 .../github-delete-20190129-215230.headers.json | 0 .../github-delete-20190129-215230.payload.json | 0 ...thub-issues-assigned-20190128-101919.headers.json | 0 ...thub-issues-assigned-20190128-101919.payload.json | 0 ...github-issues-closed-20190128-101908.headers.json | 0 ...github-issues-closed-20190128-101908.payload.json | 0 ...github-issues-opened-20190128-101904.headers.json | 0 ...github-issues-opened-20190128-101904.payload.json | 0 ...thub-issues-reopened-20190128-101912.headers.json | 0 ...thub-issues-reopened-20190128-101912.payload.json | 0 ...ub-issues-unassigned-20190128-101924.headers.json | 0 ...ub-issues-unassigned-20190128-101924.payload.json | 0 .../github/github-ping-20190128-101509.headers.json | 0 .../github/github-ping-20190128-101509.payload.json | 0 ...ull_request-assigned-20190129-215308.headers.json | 0 ...ull_request-assigned-20190129-215308.payload.json | 0 ...-pull_request-closed-20190129-215221.headers.json | 0 ...-pull_request-closed-20190129-215221.payload.json | 0 ...-pull_request-closed-20190129-215329.headers.json | 0 ...-pull_request-closed-20190129-215329.payload.json | 0 ...-pull_request-opened-20190129-215304.headers.json | 0 ...-pull_request-opened-20190129-215304.payload.json | 0 ...ull_request-reopened-20190129-215410.headers.json | 0 ...ull_request-reopened-20190129-215410.payload.json | 0 ...est-review_requested-20190130-194425.headers.json | 0 ...est-review_requested-20190130-194425.payload.json | 0 ...l_request-unassigned-20190129-215311.headers.json | 0 ...l_request-unassigned-20190129-215311.payload.json | 0 ...est_review-submitted-20190129-220000.headers.json | 0 ...est_review-submitted-20190129-220000.payload.json | 0 .../github/github-push-20190129-215221.headers.json | 0 .../github/github-push-20190129-215221.payload.json | 0 .../github/github-push-20190129-215300.headers.json | 0 .../github/github-push-20190129-215300.payload.json | 0 .../github/github-push-20190130-195825.headers.json | 0 .../github/github-push-20190130-195825.payload.json | 0 ...ub-release-published-20190130-101053.headers.json | 0 ...ub-release-published-20190130-101053.payload.json | 0 .../fixtures/google-discovery-youtube-v3.json | 0 .../test => tests}/fixtures/imgur_album_26hit.json | 0 .../test => tests}/fixtures/imgur_album_myXfq.json | 0 .../test => tests}/fixtures/imgur_album_ysj7k.json | 0 {csbot/test => tests}/fixtures/imgur_credits.json | 0 .../fixtures/imgur_gallery_HNUmA0P.json | 0 .../test => tests}/fixtures/imgur_gallery_rYRa1.json | 0 .../test => tests}/fixtures/imgur_image_jSmKOXT.json | 0 .../test => tests}/fixtures/imgur_image_ybgvNbM.json | 0 .../fixtures/imgur_invalid_album_id.json | 0 .../fixtures/imgur_invalid_api_key.json | 0 .../fixtures/imgur_invalid_gallery.json | 0 .../fixtures/imgur_invalid_image_id.json | 0 {csbot/test => tests}/fixtures/imgur_nsfw_album.json | 0 {csbot/test => tests}/fixtures/xkcd_1.json | 0 {csbot/test => tests}/fixtures/xkcd_1363.json | 0 {csbot/test => tests}/fixtures/xkcd_1506.json | 0 {csbot/test => tests}/fixtures/xkcd_2043.json | 0 {csbot/test => tests}/fixtures/xkcd_259.json | 0 {csbot/test => tests}/fixtures/xkcd_403.json | 0 {csbot/test => tests}/fixtures/xkcd_latest.json | 0 .../test => tests}/fixtures/youtube_539OnO-YImk.json | 0 .../fixtures/youtube_access_not_configured.json | 0 .../test => tests}/fixtures/youtube_fItlK6L-khc.json | 0 {csbot/test => tests}/fixtures/youtube_flibble.json | 0 .../test => tests}/fixtures/youtube_invalid_key.json | 0 .../test => tests}/fixtures/youtube_sw4hmqVPe0E.json | 0 .../test => tests}/fixtures/youtube_vZ_YpOvRd3o.json | 0 {csbot/test => tests}/test_bot.py | 0 {csbot/test => tests}/test_config.py | 2 +- {csbot/test => tests}/test_events.py | 0 {csbot/test => tests}/test_irc.py | 2 +- {csbot/test => tests}/test_plugin_auth.py | 0 {csbot/test => tests}/test_plugin_calc.py | 0 {csbot/test => tests}/test_plugin_github.py | 4 ++-- {csbot/test => tests}/test_plugin_helix.py | 0 {csbot/test => tests}/test_plugin_imgur.py | 2 +- {csbot/test => tests}/test_plugin_linkinfo.py | 0 {csbot/test => tests}/test_plugin_usertrack.py | 0 {csbot/test => tests}/test_plugin_webhook.py | 2 +- {csbot/test => tests}/test_plugin_webserver.py | 0 {csbot/test => tests}/test_plugin_whois.py | 0 {csbot/test => tests}/test_plugin_xkcd.py | 2 +- {csbot/test => tests}/test_plugin_youtube.py | 2 +- {csbot/test => tests}/test_util.py | 0 122 files changed, 21 insertions(+), 15 deletions(-) create mode 100644 setup.cfg rename {csbot => src/csbot}/__init__.py (100%) rename {csbot => src/csbot}/_rfc.py (100%) rename {csbot => src/csbot}/core.py (100%) rename {csbot => src/csbot}/events.py (100%) rename {csbot => src/csbot}/irc.py (100%) rename {csbot => src/csbot}/plugin.py (100%) rename {csbot => src/csbot}/plugins/__init__.py (100%) rename {csbot => src/csbot}/plugins/auth.py (100%) rename {csbot => src/csbot}/plugins/calc.py (100%) rename {csbot => src/csbot}/plugins/cron.py (100%) rename {csbot => src/csbot}/plugins/csyork.py (100%) rename {csbot => src/csbot}/plugins/github.py (100%) rename {csbot => src/csbot}/plugins/helix.py (100%) rename {csbot => src/csbot}/plugins/hoogle.py (100%) rename {csbot => src/csbot}/plugins/imgur.py (100%) rename {csbot => src/csbot}/plugins/last.py (100%) rename {csbot => src/csbot}/plugins/linkinfo.py (100%) rename {csbot => src/csbot}/plugins/logger.py (100%) rename {csbot => src/csbot}/plugins/mongodb.py (100%) rename {csbot => src/csbot}/plugins/termdates.py (100%) rename {csbot => src/csbot}/plugins/topic.py (100%) rename {csbot => src/csbot}/plugins/usertrack.py (100%) rename {csbot => src/csbot}/plugins/webhook.py (100%) rename {csbot => src/csbot}/plugins/webserver.py (100%) rename {csbot => src/csbot}/plugins/whois.py (100%) rename {csbot => src/csbot}/plugins/xkcd.py (100%) rename {csbot => src/csbot}/plugins/youtube.py (100%) rename {csbot => src/csbot}/plugins_broken/tell.py (100%) rename {csbot => src/csbot}/plugins_broken/users.py (100%) rename {csbot => src/csbot}/util.py (100%) rename {csbot/test => tests}/__init__.py (100%) rename {csbot/test => tests}/conftest.py (98%) rename {csbot/test => tests}/fixtures/empty_file (100%) rename {csbot/test => tests}/fixtures/github/github-create-20190129-215300.headers.json (100%) rename {csbot/test => tests}/fixtures/github/github-create-20190129-215300.payload.json (100%) rename {csbot/test => tests}/fixtures/github/github-create-20190130-101054.headers.json (100%) rename {csbot/test => tests}/fixtures/github/github-create-20190130-101054.payload.json (100%) rename {csbot/test => tests}/fixtures/github/github-delete-20190129-215230.headers.json (100%) rename {csbot/test => tests}/fixtures/github/github-delete-20190129-215230.payload.json (100%) rename {csbot/test => tests}/fixtures/github/github-issues-assigned-20190128-101919.headers.json (100%) rename {csbot/test => tests}/fixtures/github/github-issues-assigned-20190128-101919.payload.json (100%) rename {csbot/test => tests}/fixtures/github/github-issues-closed-20190128-101908.headers.json (100%) rename {csbot/test => tests}/fixtures/github/github-issues-closed-20190128-101908.payload.json (100%) rename {csbot/test => tests}/fixtures/github/github-issues-opened-20190128-101904.headers.json (100%) rename {csbot/test => tests}/fixtures/github/github-issues-opened-20190128-101904.payload.json (100%) rename {csbot/test => tests}/fixtures/github/github-issues-reopened-20190128-101912.headers.json (100%) rename {csbot/test => tests}/fixtures/github/github-issues-reopened-20190128-101912.payload.json (100%) rename {csbot/test => tests}/fixtures/github/github-issues-unassigned-20190128-101924.headers.json (100%) rename {csbot/test => tests}/fixtures/github/github-issues-unassigned-20190128-101924.payload.json (100%) rename {csbot/test => tests}/fixtures/github/github-ping-20190128-101509.headers.json (100%) rename {csbot/test => tests}/fixtures/github/github-ping-20190128-101509.payload.json (100%) rename {csbot/test => tests}/fixtures/github/github-pull_request-assigned-20190129-215308.headers.json (100%) rename {csbot/test => tests}/fixtures/github/github-pull_request-assigned-20190129-215308.payload.json (100%) rename {csbot/test => tests}/fixtures/github/github-pull_request-closed-20190129-215221.headers.json (100%) rename {csbot/test => tests}/fixtures/github/github-pull_request-closed-20190129-215221.payload.json (100%) rename {csbot/test => tests}/fixtures/github/github-pull_request-closed-20190129-215329.headers.json (100%) rename {csbot/test => tests}/fixtures/github/github-pull_request-closed-20190129-215329.payload.json (100%) rename {csbot/test => tests}/fixtures/github/github-pull_request-opened-20190129-215304.headers.json (100%) rename {csbot/test => tests}/fixtures/github/github-pull_request-opened-20190129-215304.payload.json (100%) rename {csbot/test => tests}/fixtures/github/github-pull_request-reopened-20190129-215410.headers.json (100%) rename {csbot/test => tests}/fixtures/github/github-pull_request-reopened-20190129-215410.payload.json (100%) rename {csbot/test => tests}/fixtures/github/github-pull_request-review_requested-20190130-194425.headers.json (100%) rename {csbot/test => tests}/fixtures/github/github-pull_request-review_requested-20190130-194425.payload.json (100%) rename {csbot/test => tests}/fixtures/github/github-pull_request-unassigned-20190129-215311.headers.json (100%) rename {csbot/test => tests}/fixtures/github/github-pull_request-unassigned-20190129-215311.payload.json (100%) rename {csbot/test => tests}/fixtures/github/github-pull_request_review-submitted-20190129-220000.headers.json (100%) rename {csbot/test => tests}/fixtures/github/github-pull_request_review-submitted-20190129-220000.payload.json (100%) rename {csbot/test => tests}/fixtures/github/github-push-20190129-215221.headers.json (100%) rename {csbot/test => tests}/fixtures/github/github-push-20190129-215221.payload.json (100%) rename {csbot/test => tests}/fixtures/github/github-push-20190129-215300.headers.json (100%) rename {csbot/test => tests}/fixtures/github/github-push-20190129-215300.payload.json (100%) rename {csbot/test => tests}/fixtures/github/github-push-20190130-195825.headers.json (100%) rename {csbot/test => tests}/fixtures/github/github-push-20190130-195825.payload.json (100%) rename {csbot/test => tests}/fixtures/github/github-release-published-20190130-101053.headers.json (100%) rename {csbot/test => tests}/fixtures/github/github-release-published-20190130-101053.payload.json (100%) rename {csbot/test => tests}/fixtures/google-discovery-youtube-v3.json (100%) rename {csbot/test => tests}/fixtures/imgur_album_26hit.json (100%) rename {csbot/test => tests}/fixtures/imgur_album_myXfq.json (100%) rename {csbot/test => tests}/fixtures/imgur_album_ysj7k.json (100%) rename {csbot/test => tests}/fixtures/imgur_credits.json (100%) rename {csbot/test => tests}/fixtures/imgur_gallery_HNUmA0P.json (100%) rename {csbot/test => tests}/fixtures/imgur_gallery_rYRa1.json (100%) rename {csbot/test => tests}/fixtures/imgur_image_jSmKOXT.json (100%) rename {csbot/test => tests}/fixtures/imgur_image_ybgvNbM.json (100%) rename {csbot/test => tests}/fixtures/imgur_invalid_album_id.json (100%) rename {csbot/test => tests}/fixtures/imgur_invalid_api_key.json (100%) rename {csbot/test => tests}/fixtures/imgur_invalid_gallery.json (100%) rename {csbot/test => tests}/fixtures/imgur_invalid_image_id.json (100%) rename {csbot/test => tests}/fixtures/imgur_nsfw_album.json (100%) rename {csbot/test => tests}/fixtures/xkcd_1.json (100%) rename {csbot/test => tests}/fixtures/xkcd_1363.json (100%) rename {csbot/test => tests}/fixtures/xkcd_1506.json (100%) rename {csbot/test => tests}/fixtures/xkcd_2043.json (100%) rename {csbot/test => tests}/fixtures/xkcd_259.json (100%) rename {csbot/test => tests}/fixtures/xkcd_403.json (100%) rename {csbot/test => tests}/fixtures/xkcd_latest.json (100%) rename {csbot/test => tests}/fixtures/youtube_539OnO-YImk.json (100%) rename {csbot/test => tests}/fixtures/youtube_access_not_configured.json (100%) rename {csbot/test => tests}/fixtures/youtube_fItlK6L-khc.json (100%) rename {csbot/test => tests}/fixtures/youtube_flibble.json (100%) rename {csbot/test => tests}/fixtures/youtube_invalid_key.json (100%) rename {csbot/test => tests}/fixtures/youtube_sw4hmqVPe0E.json (100%) rename {csbot/test => tests}/fixtures/youtube_vZ_YpOvRd3o.json (100%) rename {csbot/test => tests}/test_bot.py (100%) rename {csbot/test => tests}/test_config.py (98%) rename {csbot/test => tests}/test_events.py (100%) rename {csbot/test => tests}/test_irc.py (99%) rename {csbot/test => tests}/test_plugin_auth.py (100%) rename {csbot/test => tests}/test_plugin_calc.py (100%) rename {csbot/test => tests}/test_plugin_github.py (99%) rename {csbot/test => tests}/test_plugin_helix.py (100%) rename {csbot/test => tests}/test_plugin_imgur.py (99%) rename {csbot/test => tests}/test_plugin_linkinfo.py (100%) rename {csbot/test => tests}/test_plugin_usertrack.py (100%) rename {csbot/test => tests}/test_plugin_webhook.py (97%) rename {csbot/test => tests}/test_plugin_webserver.py (100%) rename {csbot/test => tests}/test_plugin_whois.py (100%) rename {csbot/test => tests}/test_plugin_xkcd.py (99%) rename {csbot/test => tests}/test_plugin_youtube.py (99%) rename {csbot/test => tests}/test_util.py (100%) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..39baf331 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[pytest] +testpaths = tests/ diff --git a/setup.py b/setup.py index 1addecc7..b8598f7b 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,9 @@ from distutils.core import setup -setup(name='csbot', - version='0.1', - packages=['csbot', 'csbot.plugins'], - ) + + +setup( + name='csbot', + version='0.1', + packages=['csbot', 'csbot.plugins'], + package_dir={'': 'src'}, +) diff --git a/csbot/__init__.py b/src/csbot/__init__.py similarity index 100% rename from csbot/__init__.py rename to src/csbot/__init__.py diff --git a/csbot/_rfc.py b/src/csbot/_rfc.py similarity index 100% rename from csbot/_rfc.py rename to src/csbot/_rfc.py diff --git a/csbot/core.py b/src/csbot/core.py similarity index 100% rename from csbot/core.py rename to src/csbot/core.py diff --git a/csbot/events.py b/src/csbot/events.py similarity index 100% rename from csbot/events.py rename to src/csbot/events.py diff --git a/csbot/irc.py b/src/csbot/irc.py similarity index 100% rename from csbot/irc.py rename to src/csbot/irc.py diff --git a/csbot/plugin.py b/src/csbot/plugin.py similarity index 100% rename from csbot/plugin.py rename to src/csbot/plugin.py diff --git a/csbot/plugins/__init__.py b/src/csbot/plugins/__init__.py similarity index 100% rename from csbot/plugins/__init__.py rename to src/csbot/plugins/__init__.py diff --git a/csbot/plugins/auth.py b/src/csbot/plugins/auth.py similarity index 100% rename from csbot/plugins/auth.py rename to src/csbot/plugins/auth.py diff --git a/csbot/plugins/calc.py b/src/csbot/plugins/calc.py similarity index 100% rename from csbot/plugins/calc.py rename to src/csbot/plugins/calc.py diff --git a/csbot/plugins/cron.py b/src/csbot/plugins/cron.py similarity index 100% rename from csbot/plugins/cron.py rename to src/csbot/plugins/cron.py diff --git a/csbot/plugins/csyork.py b/src/csbot/plugins/csyork.py similarity index 100% rename from csbot/plugins/csyork.py rename to src/csbot/plugins/csyork.py diff --git a/csbot/plugins/github.py b/src/csbot/plugins/github.py similarity index 100% rename from csbot/plugins/github.py rename to src/csbot/plugins/github.py diff --git a/csbot/plugins/helix.py b/src/csbot/plugins/helix.py similarity index 100% rename from csbot/plugins/helix.py rename to src/csbot/plugins/helix.py diff --git a/csbot/plugins/hoogle.py b/src/csbot/plugins/hoogle.py similarity index 100% rename from csbot/plugins/hoogle.py rename to src/csbot/plugins/hoogle.py diff --git a/csbot/plugins/imgur.py b/src/csbot/plugins/imgur.py similarity index 100% rename from csbot/plugins/imgur.py rename to src/csbot/plugins/imgur.py diff --git a/csbot/plugins/last.py b/src/csbot/plugins/last.py similarity index 100% rename from csbot/plugins/last.py rename to src/csbot/plugins/last.py diff --git a/csbot/plugins/linkinfo.py b/src/csbot/plugins/linkinfo.py similarity index 100% rename from csbot/plugins/linkinfo.py rename to src/csbot/plugins/linkinfo.py diff --git a/csbot/plugins/logger.py b/src/csbot/plugins/logger.py similarity index 100% rename from csbot/plugins/logger.py rename to src/csbot/plugins/logger.py diff --git a/csbot/plugins/mongodb.py b/src/csbot/plugins/mongodb.py similarity index 100% rename from csbot/plugins/mongodb.py rename to src/csbot/plugins/mongodb.py diff --git a/csbot/plugins/termdates.py b/src/csbot/plugins/termdates.py similarity index 100% rename from csbot/plugins/termdates.py rename to src/csbot/plugins/termdates.py diff --git a/csbot/plugins/topic.py b/src/csbot/plugins/topic.py similarity index 100% rename from csbot/plugins/topic.py rename to src/csbot/plugins/topic.py diff --git a/csbot/plugins/usertrack.py b/src/csbot/plugins/usertrack.py similarity index 100% rename from csbot/plugins/usertrack.py rename to src/csbot/plugins/usertrack.py diff --git a/csbot/plugins/webhook.py b/src/csbot/plugins/webhook.py similarity index 100% rename from csbot/plugins/webhook.py rename to src/csbot/plugins/webhook.py diff --git a/csbot/plugins/webserver.py b/src/csbot/plugins/webserver.py similarity index 100% rename from csbot/plugins/webserver.py rename to src/csbot/plugins/webserver.py diff --git a/csbot/plugins/whois.py b/src/csbot/plugins/whois.py similarity index 100% rename from csbot/plugins/whois.py rename to src/csbot/plugins/whois.py diff --git a/csbot/plugins/xkcd.py b/src/csbot/plugins/xkcd.py similarity index 100% rename from csbot/plugins/xkcd.py rename to src/csbot/plugins/xkcd.py diff --git a/csbot/plugins/youtube.py b/src/csbot/plugins/youtube.py similarity index 100% rename from csbot/plugins/youtube.py rename to src/csbot/plugins/youtube.py diff --git a/csbot/plugins_broken/tell.py b/src/csbot/plugins_broken/tell.py similarity index 100% rename from csbot/plugins_broken/tell.py rename to src/csbot/plugins_broken/tell.py diff --git a/csbot/plugins_broken/users.py b/src/csbot/plugins_broken/users.py similarity index 100% rename from csbot/plugins_broken/users.py rename to src/csbot/plugins_broken/users.py diff --git a/csbot/util.py b/src/csbot/util.py similarity index 100% rename from csbot/util.py rename to src/csbot/util.py diff --git a/csbot/test/__init__.py b/tests/__init__.py similarity index 100% rename from csbot/test/__init__.py rename to tests/__init__.py diff --git a/csbot/test/conftest.py b/tests/conftest.py similarity index 98% rename from csbot/test/conftest.py rename to tests/conftest.py index 145e073f..0371c8a1 100644 --- a/csbot/test/conftest.py +++ b/tests/conftest.py @@ -8,9 +8,9 @@ import responses as responses_ from aioresponses import aioresponses as aioresponses_ -from csbot import test from csbot.irc import IRCClient from csbot.core import Bot +from . import mock_open_connection @pytest.fixture @@ -55,7 +55,7 @@ async def irc_client(request, event_loop, irc_client_class, pre_irc_client, irc_ else: client = irc_client_class(loop=event_loop, **irc_client_config) # Connect fake stream reader/writer (for tests that don't need the read loop) - with test.mock_open_connection(): + with mock_open_connection(): await client.connect() # Mock all the things! @@ -138,7 +138,7 @@ async def run_client(event_loop, irc_client_helper): ... await irc_client_helper.receive_bytes(b":nick!user@host PRIVMSG #channel :hello\r\n") ... irc_client_helper.assert_sent('PRIVMSG #channel :what do you mean, hello?') """ - with test.mock_open_connection(): + with mock_open_connection(): # Start the client run_fut = event_loop.create_task(irc_client_helper.client.run()) await irc_client_helper.client.connected.wait() diff --git a/csbot/test/fixtures/empty_file b/tests/fixtures/empty_file similarity index 100% rename from csbot/test/fixtures/empty_file rename to tests/fixtures/empty_file diff --git a/csbot/test/fixtures/github/github-create-20190129-215300.headers.json b/tests/fixtures/github/github-create-20190129-215300.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-create-20190129-215300.headers.json rename to tests/fixtures/github/github-create-20190129-215300.headers.json diff --git a/csbot/test/fixtures/github/github-create-20190129-215300.payload.json b/tests/fixtures/github/github-create-20190129-215300.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-create-20190129-215300.payload.json rename to tests/fixtures/github/github-create-20190129-215300.payload.json diff --git a/csbot/test/fixtures/github/github-create-20190130-101054.headers.json b/tests/fixtures/github/github-create-20190130-101054.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-create-20190130-101054.headers.json rename to tests/fixtures/github/github-create-20190130-101054.headers.json diff --git a/csbot/test/fixtures/github/github-create-20190130-101054.payload.json b/tests/fixtures/github/github-create-20190130-101054.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-create-20190130-101054.payload.json rename to tests/fixtures/github/github-create-20190130-101054.payload.json diff --git a/csbot/test/fixtures/github/github-delete-20190129-215230.headers.json b/tests/fixtures/github/github-delete-20190129-215230.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-delete-20190129-215230.headers.json rename to tests/fixtures/github/github-delete-20190129-215230.headers.json diff --git a/csbot/test/fixtures/github/github-delete-20190129-215230.payload.json b/tests/fixtures/github/github-delete-20190129-215230.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-delete-20190129-215230.payload.json rename to tests/fixtures/github/github-delete-20190129-215230.payload.json diff --git a/csbot/test/fixtures/github/github-issues-assigned-20190128-101919.headers.json b/tests/fixtures/github/github-issues-assigned-20190128-101919.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-issues-assigned-20190128-101919.headers.json rename to tests/fixtures/github/github-issues-assigned-20190128-101919.headers.json diff --git a/csbot/test/fixtures/github/github-issues-assigned-20190128-101919.payload.json b/tests/fixtures/github/github-issues-assigned-20190128-101919.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-issues-assigned-20190128-101919.payload.json rename to tests/fixtures/github/github-issues-assigned-20190128-101919.payload.json diff --git a/csbot/test/fixtures/github/github-issues-closed-20190128-101908.headers.json b/tests/fixtures/github/github-issues-closed-20190128-101908.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-issues-closed-20190128-101908.headers.json rename to tests/fixtures/github/github-issues-closed-20190128-101908.headers.json diff --git a/csbot/test/fixtures/github/github-issues-closed-20190128-101908.payload.json b/tests/fixtures/github/github-issues-closed-20190128-101908.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-issues-closed-20190128-101908.payload.json rename to tests/fixtures/github/github-issues-closed-20190128-101908.payload.json diff --git a/csbot/test/fixtures/github/github-issues-opened-20190128-101904.headers.json b/tests/fixtures/github/github-issues-opened-20190128-101904.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-issues-opened-20190128-101904.headers.json rename to tests/fixtures/github/github-issues-opened-20190128-101904.headers.json diff --git a/csbot/test/fixtures/github/github-issues-opened-20190128-101904.payload.json b/tests/fixtures/github/github-issues-opened-20190128-101904.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-issues-opened-20190128-101904.payload.json rename to tests/fixtures/github/github-issues-opened-20190128-101904.payload.json diff --git a/csbot/test/fixtures/github/github-issues-reopened-20190128-101912.headers.json b/tests/fixtures/github/github-issues-reopened-20190128-101912.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-issues-reopened-20190128-101912.headers.json rename to tests/fixtures/github/github-issues-reopened-20190128-101912.headers.json diff --git a/csbot/test/fixtures/github/github-issues-reopened-20190128-101912.payload.json b/tests/fixtures/github/github-issues-reopened-20190128-101912.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-issues-reopened-20190128-101912.payload.json rename to tests/fixtures/github/github-issues-reopened-20190128-101912.payload.json diff --git a/csbot/test/fixtures/github/github-issues-unassigned-20190128-101924.headers.json b/tests/fixtures/github/github-issues-unassigned-20190128-101924.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-issues-unassigned-20190128-101924.headers.json rename to tests/fixtures/github/github-issues-unassigned-20190128-101924.headers.json diff --git a/csbot/test/fixtures/github/github-issues-unassigned-20190128-101924.payload.json b/tests/fixtures/github/github-issues-unassigned-20190128-101924.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-issues-unassigned-20190128-101924.payload.json rename to tests/fixtures/github/github-issues-unassigned-20190128-101924.payload.json diff --git a/csbot/test/fixtures/github/github-ping-20190128-101509.headers.json b/tests/fixtures/github/github-ping-20190128-101509.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-ping-20190128-101509.headers.json rename to tests/fixtures/github/github-ping-20190128-101509.headers.json diff --git a/csbot/test/fixtures/github/github-ping-20190128-101509.payload.json b/tests/fixtures/github/github-ping-20190128-101509.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-ping-20190128-101509.payload.json rename to tests/fixtures/github/github-ping-20190128-101509.payload.json diff --git a/csbot/test/fixtures/github/github-pull_request-assigned-20190129-215308.headers.json b/tests/fixtures/github/github-pull_request-assigned-20190129-215308.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-assigned-20190129-215308.headers.json rename to tests/fixtures/github/github-pull_request-assigned-20190129-215308.headers.json diff --git a/csbot/test/fixtures/github/github-pull_request-assigned-20190129-215308.payload.json b/tests/fixtures/github/github-pull_request-assigned-20190129-215308.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-assigned-20190129-215308.payload.json rename to tests/fixtures/github/github-pull_request-assigned-20190129-215308.payload.json diff --git a/csbot/test/fixtures/github/github-pull_request-closed-20190129-215221.headers.json b/tests/fixtures/github/github-pull_request-closed-20190129-215221.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-closed-20190129-215221.headers.json rename to tests/fixtures/github/github-pull_request-closed-20190129-215221.headers.json diff --git a/csbot/test/fixtures/github/github-pull_request-closed-20190129-215221.payload.json b/tests/fixtures/github/github-pull_request-closed-20190129-215221.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-closed-20190129-215221.payload.json rename to tests/fixtures/github/github-pull_request-closed-20190129-215221.payload.json diff --git a/csbot/test/fixtures/github/github-pull_request-closed-20190129-215329.headers.json b/tests/fixtures/github/github-pull_request-closed-20190129-215329.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-closed-20190129-215329.headers.json rename to tests/fixtures/github/github-pull_request-closed-20190129-215329.headers.json diff --git a/csbot/test/fixtures/github/github-pull_request-closed-20190129-215329.payload.json b/tests/fixtures/github/github-pull_request-closed-20190129-215329.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-closed-20190129-215329.payload.json rename to tests/fixtures/github/github-pull_request-closed-20190129-215329.payload.json diff --git a/csbot/test/fixtures/github/github-pull_request-opened-20190129-215304.headers.json b/tests/fixtures/github/github-pull_request-opened-20190129-215304.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-opened-20190129-215304.headers.json rename to tests/fixtures/github/github-pull_request-opened-20190129-215304.headers.json diff --git a/csbot/test/fixtures/github/github-pull_request-opened-20190129-215304.payload.json b/tests/fixtures/github/github-pull_request-opened-20190129-215304.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-opened-20190129-215304.payload.json rename to tests/fixtures/github/github-pull_request-opened-20190129-215304.payload.json diff --git a/csbot/test/fixtures/github/github-pull_request-reopened-20190129-215410.headers.json b/tests/fixtures/github/github-pull_request-reopened-20190129-215410.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-reopened-20190129-215410.headers.json rename to tests/fixtures/github/github-pull_request-reopened-20190129-215410.headers.json diff --git a/csbot/test/fixtures/github/github-pull_request-reopened-20190129-215410.payload.json b/tests/fixtures/github/github-pull_request-reopened-20190129-215410.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-reopened-20190129-215410.payload.json rename to tests/fixtures/github/github-pull_request-reopened-20190129-215410.payload.json diff --git a/csbot/test/fixtures/github/github-pull_request-review_requested-20190130-194425.headers.json b/tests/fixtures/github/github-pull_request-review_requested-20190130-194425.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-review_requested-20190130-194425.headers.json rename to tests/fixtures/github/github-pull_request-review_requested-20190130-194425.headers.json diff --git a/csbot/test/fixtures/github/github-pull_request-review_requested-20190130-194425.payload.json b/tests/fixtures/github/github-pull_request-review_requested-20190130-194425.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-review_requested-20190130-194425.payload.json rename to tests/fixtures/github/github-pull_request-review_requested-20190130-194425.payload.json diff --git a/csbot/test/fixtures/github/github-pull_request-unassigned-20190129-215311.headers.json b/tests/fixtures/github/github-pull_request-unassigned-20190129-215311.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-unassigned-20190129-215311.headers.json rename to tests/fixtures/github/github-pull_request-unassigned-20190129-215311.headers.json diff --git a/csbot/test/fixtures/github/github-pull_request-unassigned-20190129-215311.payload.json b/tests/fixtures/github/github-pull_request-unassigned-20190129-215311.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request-unassigned-20190129-215311.payload.json rename to tests/fixtures/github/github-pull_request-unassigned-20190129-215311.payload.json diff --git a/csbot/test/fixtures/github/github-pull_request_review-submitted-20190129-220000.headers.json b/tests/fixtures/github/github-pull_request_review-submitted-20190129-220000.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request_review-submitted-20190129-220000.headers.json rename to tests/fixtures/github/github-pull_request_review-submitted-20190129-220000.headers.json diff --git a/csbot/test/fixtures/github/github-pull_request_review-submitted-20190129-220000.payload.json b/tests/fixtures/github/github-pull_request_review-submitted-20190129-220000.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-pull_request_review-submitted-20190129-220000.payload.json rename to tests/fixtures/github/github-pull_request_review-submitted-20190129-220000.payload.json diff --git a/csbot/test/fixtures/github/github-push-20190129-215221.headers.json b/tests/fixtures/github/github-push-20190129-215221.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-push-20190129-215221.headers.json rename to tests/fixtures/github/github-push-20190129-215221.headers.json diff --git a/csbot/test/fixtures/github/github-push-20190129-215221.payload.json b/tests/fixtures/github/github-push-20190129-215221.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-push-20190129-215221.payload.json rename to tests/fixtures/github/github-push-20190129-215221.payload.json diff --git a/csbot/test/fixtures/github/github-push-20190129-215300.headers.json b/tests/fixtures/github/github-push-20190129-215300.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-push-20190129-215300.headers.json rename to tests/fixtures/github/github-push-20190129-215300.headers.json diff --git a/csbot/test/fixtures/github/github-push-20190129-215300.payload.json b/tests/fixtures/github/github-push-20190129-215300.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-push-20190129-215300.payload.json rename to tests/fixtures/github/github-push-20190129-215300.payload.json diff --git a/csbot/test/fixtures/github/github-push-20190130-195825.headers.json b/tests/fixtures/github/github-push-20190130-195825.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-push-20190130-195825.headers.json rename to tests/fixtures/github/github-push-20190130-195825.headers.json diff --git a/csbot/test/fixtures/github/github-push-20190130-195825.payload.json b/tests/fixtures/github/github-push-20190130-195825.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-push-20190130-195825.payload.json rename to tests/fixtures/github/github-push-20190130-195825.payload.json diff --git a/csbot/test/fixtures/github/github-release-published-20190130-101053.headers.json b/tests/fixtures/github/github-release-published-20190130-101053.headers.json similarity index 100% rename from csbot/test/fixtures/github/github-release-published-20190130-101053.headers.json rename to tests/fixtures/github/github-release-published-20190130-101053.headers.json diff --git a/csbot/test/fixtures/github/github-release-published-20190130-101053.payload.json b/tests/fixtures/github/github-release-published-20190130-101053.payload.json similarity index 100% rename from csbot/test/fixtures/github/github-release-published-20190130-101053.payload.json rename to tests/fixtures/github/github-release-published-20190130-101053.payload.json diff --git a/csbot/test/fixtures/google-discovery-youtube-v3.json b/tests/fixtures/google-discovery-youtube-v3.json similarity index 100% rename from csbot/test/fixtures/google-discovery-youtube-v3.json rename to tests/fixtures/google-discovery-youtube-v3.json diff --git a/csbot/test/fixtures/imgur_album_26hit.json b/tests/fixtures/imgur_album_26hit.json similarity index 100% rename from csbot/test/fixtures/imgur_album_26hit.json rename to tests/fixtures/imgur_album_26hit.json diff --git a/csbot/test/fixtures/imgur_album_myXfq.json b/tests/fixtures/imgur_album_myXfq.json similarity index 100% rename from csbot/test/fixtures/imgur_album_myXfq.json rename to tests/fixtures/imgur_album_myXfq.json diff --git a/csbot/test/fixtures/imgur_album_ysj7k.json b/tests/fixtures/imgur_album_ysj7k.json similarity index 100% rename from csbot/test/fixtures/imgur_album_ysj7k.json rename to tests/fixtures/imgur_album_ysj7k.json diff --git a/csbot/test/fixtures/imgur_credits.json b/tests/fixtures/imgur_credits.json similarity index 100% rename from csbot/test/fixtures/imgur_credits.json rename to tests/fixtures/imgur_credits.json diff --git a/csbot/test/fixtures/imgur_gallery_HNUmA0P.json b/tests/fixtures/imgur_gallery_HNUmA0P.json similarity index 100% rename from csbot/test/fixtures/imgur_gallery_HNUmA0P.json rename to tests/fixtures/imgur_gallery_HNUmA0P.json diff --git a/csbot/test/fixtures/imgur_gallery_rYRa1.json b/tests/fixtures/imgur_gallery_rYRa1.json similarity index 100% rename from csbot/test/fixtures/imgur_gallery_rYRa1.json rename to tests/fixtures/imgur_gallery_rYRa1.json diff --git a/csbot/test/fixtures/imgur_image_jSmKOXT.json b/tests/fixtures/imgur_image_jSmKOXT.json similarity index 100% rename from csbot/test/fixtures/imgur_image_jSmKOXT.json rename to tests/fixtures/imgur_image_jSmKOXT.json diff --git a/csbot/test/fixtures/imgur_image_ybgvNbM.json b/tests/fixtures/imgur_image_ybgvNbM.json similarity index 100% rename from csbot/test/fixtures/imgur_image_ybgvNbM.json rename to tests/fixtures/imgur_image_ybgvNbM.json diff --git a/csbot/test/fixtures/imgur_invalid_album_id.json b/tests/fixtures/imgur_invalid_album_id.json similarity index 100% rename from csbot/test/fixtures/imgur_invalid_album_id.json rename to tests/fixtures/imgur_invalid_album_id.json diff --git a/csbot/test/fixtures/imgur_invalid_api_key.json b/tests/fixtures/imgur_invalid_api_key.json similarity index 100% rename from csbot/test/fixtures/imgur_invalid_api_key.json rename to tests/fixtures/imgur_invalid_api_key.json diff --git a/csbot/test/fixtures/imgur_invalid_gallery.json b/tests/fixtures/imgur_invalid_gallery.json similarity index 100% rename from csbot/test/fixtures/imgur_invalid_gallery.json rename to tests/fixtures/imgur_invalid_gallery.json diff --git a/csbot/test/fixtures/imgur_invalid_image_id.json b/tests/fixtures/imgur_invalid_image_id.json similarity index 100% rename from csbot/test/fixtures/imgur_invalid_image_id.json rename to tests/fixtures/imgur_invalid_image_id.json diff --git a/csbot/test/fixtures/imgur_nsfw_album.json b/tests/fixtures/imgur_nsfw_album.json similarity index 100% rename from csbot/test/fixtures/imgur_nsfw_album.json rename to tests/fixtures/imgur_nsfw_album.json diff --git a/csbot/test/fixtures/xkcd_1.json b/tests/fixtures/xkcd_1.json similarity index 100% rename from csbot/test/fixtures/xkcd_1.json rename to tests/fixtures/xkcd_1.json diff --git a/csbot/test/fixtures/xkcd_1363.json b/tests/fixtures/xkcd_1363.json similarity index 100% rename from csbot/test/fixtures/xkcd_1363.json rename to tests/fixtures/xkcd_1363.json diff --git a/csbot/test/fixtures/xkcd_1506.json b/tests/fixtures/xkcd_1506.json similarity index 100% rename from csbot/test/fixtures/xkcd_1506.json rename to tests/fixtures/xkcd_1506.json diff --git a/csbot/test/fixtures/xkcd_2043.json b/tests/fixtures/xkcd_2043.json similarity index 100% rename from csbot/test/fixtures/xkcd_2043.json rename to tests/fixtures/xkcd_2043.json diff --git a/csbot/test/fixtures/xkcd_259.json b/tests/fixtures/xkcd_259.json similarity index 100% rename from csbot/test/fixtures/xkcd_259.json rename to tests/fixtures/xkcd_259.json diff --git a/csbot/test/fixtures/xkcd_403.json b/tests/fixtures/xkcd_403.json similarity index 100% rename from csbot/test/fixtures/xkcd_403.json rename to tests/fixtures/xkcd_403.json diff --git a/csbot/test/fixtures/xkcd_latest.json b/tests/fixtures/xkcd_latest.json similarity index 100% rename from csbot/test/fixtures/xkcd_latest.json rename to tests/fixtures/xkcd_latest.json diff --git a/csbot/test/fixtures/youtube_539OnO-YImk.json b/tests/fixtures/youtube_539OnO-YImk.json similarity index 100% rename from csbot/test/fixtures/youtube_539OnO-YImk.json rename to tests/fixtures/youtube_539OnO-YImk.json diff --git a/csbot/test/fixtures/youtube_access_not_configured.json b/tests/fixtures/youtube_access_not_configured.json similarity index 100% rename from csbot/test/fixtures/youtube_access_not_configured.json rename to tests/fixtures/youtube_access_not_configured.json diff --git a/csbot/test/fixtures/youtube_fItlK6L-khc.json b/tests/fixtures/youtube_fItlK6L-khc.json similarity index 100% rename from csbot/test/fixtures/youtube_fItlK6L-khc.json rename to tests/fixtures/youtube_fItlK6L-khc.json diff --git a/csbot/test/fixtures/youtube_flibble.json b/tests/fixtures/youtube_flibble.json similarity index 100% rename from csbot/test/fixtures/youtube_flibble.json rename to tests/fixtures/youtube_flibble.json diff --git a/csbot/test/fixtures/youtube_invalid_key.json b/tests/fixtures/youtube_invalid_key.json similarity index 100% rename from csbot/test/fixtures/youtube_invalid_key.json rename to tests/fixtures/youtube_invalid_key.json diff --git a/csbot/test/fixtures/youtube_sw4hmqVPe0E.json b/tests/fixtures/youtube_sw4hmqVPe0E.json similarity index 100% rename from csbot/test/fixtures/youtube_sw4hmqVPe0E.json rename to tests/fixtures/youtube_sw4hmqVPe0E.json diff --git a/csbot/test/fixtures/youtube_vZ_YpOvRd3o.json b/tests/fixtures/youtube_vZ_YpOvRd3o.json similarity index 100% rename from csbot/test/fixtures/youtube_vZ_YpOvRd3o.json rename to tests/fixtures/youtube_vZ_YpOvRd3o.json diff --git a/csbot/test/test_bot.py b/tests/test_bot.py similarity index 100% rename from csbot/test/test_bot.py rename to tests/test_bot.py diff --git a/csbot/test/test_config.py b/tests/test_config.py similarity index 98% rename from csbot/test/test_config.py rename to tests/test_config.py index b7521a74..d94109cd 100644 --- a/csbot/test/test_config.py +++ b/tests/test_config.py @@ -1,6 +1,6 @@ import pytest -from csbot.test import TempEnvVars +from . import TempEnvVars import csbot.plugin diff --git a/csbot/test/test_events.py b/tests/test_events.py similarity index 100% rename from csbot/test/test_events.py rename to tests/test_events.py diff --git a/csbot/test/test_irc.py b/tests/test_irc.py similarity index 99% rename from csbot/test/test_irc.py rename to tests/test_irc.py index 2a652885..0990b94e 100644 --- a/csbot/test/test_irc.py +++ b/tests/test_irc.py @@ -3,7 +3,7 @@ import pytest -from csbot.test import mock_open_connection, mock_open_connection_paused +from . import mock_open_connection, mock_open_connection_paused from csbot.irc import IRCMessage, IRCParseError, IRCUser diff --git a/csbot/test/test_plugin_auth.py b/tests/test_plugin_auth.py similarity index 100% rename from csbot/test/test_plugin_auth.py rename to tests/test_plugin_auth.py diff --git a/csbot/test/test_plugin_calc.py b/tests/test_plugin_calc.py similarity index 100% rename from csbot/test/test_plugin_calc.py rename to tests/test_plugin_calc.py diff --git a/csbot/test/test_plugin_github.py b/tests/test_plugin_github.py similarity index 99% rename from csbot/test/test_plugin_github.py rename to tests/test_plugin_github.py index 490d625e..97e98d84 100644 --- a/csbot/test/test_plugin_github.py +++ b/tests/test_plugin_github.py @@ -4,8 +4,8 @@ import asynctest from csbot import core -from csbot.test import read_fixture_file -from csbot.test.test_plugin_webserver import WebServer +from . import read_fixture_file +from .test_plugin_webserver import WebServer class Bot(core.Bot): diff --git a/csbot/test/test_plugin_helix.py b/tests/test_plugin_helix.py similarity index 100% rename from csbot/test/test_plugin_helix.py rename to tests/test_plugin_helix.py diff --git a/csbot/test/test_plugin_imgur.py b/tests/test_plugin_imgur.py similarity index 99% rename from csbot/test/test_plugin_imgur.py rename to tests/test_plugin_imgur.py index f92e47ea..bc26f4fc 100644 --- a/csbot/test/test_plugin_imgur.py +++ b/tests/test_plugin_imgur.py @@ -1,6 +1,6 @@ import pytest -from csbot.test import read_fixture_file +from . import read_fixture_file test_cases = [ diff --git a/csbot/test/test_plugin_linkinfo.py b/tests/test_plugin_linkinfo.py similarity index 100% rename from csbot/test/test_plugin_linkinfo.py rename to tests/test_plugin_linkinfo.py diff --git a/csbot/test/test_plugin_usertrack.py b/tests/test_plugin_usertrack.py similarity index 100% rename from csbot/test/test_plugin_usertrack.py rename to tests/test_plugin_usertrack.py diff --git a/csbot/test/test_plugin_webhook.py b/tests/test_plugin_webhook.py similarity index 97% rename from csbot/test/test_plugin_webhook.py rename to tests/test_plugin_webhook.py index 68a4c31b..405ece45 100644 --- a/csbot/test/test_plugin_webhook.py +++ b/tests/test_plugin_webhook.py @@ -4,7 +4,7 @@ from csbot import core from csbot.plugin import Plugin -from csbot.test.test_plugin_webserver import WebServer +from .test_plugin_webserver import WebServer class WebhookTest(Plugin): diff --git a/csbot/test/test_plugin_webserver.py b/tests/test_plugin_webserver.py similarity index 100% rename from csbot/test/test_plugin_webserver.py rename to tests/test_plugin_webserver.py diff --git a/csbot/test/test_plugin_whois.py b/tests/test_plugin_whois.py similarity index 100% rename from csbot/test/test_plugin_whois.py rename to tests/test_plugin_whois.py diff --git a/csbot/test/test_plugin_xkcd.py b/tests/test_plugin_xkcd.py similarity index 99% rename from csbot/test/test_plugin_xkcd.py rename to tests/test_plugin_xkcd.py index 4863a340..1b52457b 100644 --- a/csbot/test/test_plugin_xkcd.py +++ b/tests/test_plugin_xkcd.py @@ -2,7 +2,7 @@ import pytest -from csbot.test import read_fixture_file +from . import read_fixture_file #: Tests are (number, url, content-type, fixture, expected) diff --git a/csbot/test/test_plugin_youtube.py b/tests/test_plugin_youtube.py similarity index 99% rename from csbot/test/test_plugin_youtube.py rename to tests/test_plugin_youtube.py index 8469fdd5..9d4d180c 100644 --- a/csbot/test/test_plugin_youtube.py +++ b/tests/test_plugin_youtube.py @@ -3,7 +3,7 @@ import pytest import urllib.parse as urlparse -from csbot.test import read_fixture_file +from . import read_fixture_file from csbot.plugins.youtube import YoutubeError diff --git a/csbot/test/test_util.py b/tests/test_util.py similarity index 100% rename from csbot/test/test_util.py rename to tests/test_util.py From c4d9b7ac1ff2bb5bb53ed4059d8ccecd093f1940 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Fri, 7 Jun 2019 15:45:40 +0100 Subject: [PATCH 54/66] Install as package, create `csbot` entrypoint --- README.rst | 3 ++- run_csbot.py | 3 --- setup.py | 12 ++++++++++++ src/csbot/__init__.py | 5 ++++- 4 files changed, 18 insertions(+), 5 deletions(-) delete mode 100755 run_csbot.py diff --git a/README.rst b/README.rst index db6afff7..bc2f4bb9 100644 --- a/README.rst +++ b/README.rst @@ -14,7 +14,8 @@ and running [1]_:: $ python3 -m venv venv3 $ source venv3/bin/activate $ pip install -r requirements.txt - $ ./run_csbot.py --help + $ pip install -e . + $ csbot --help Look at ``csbot.deploy.cfg`` for an example of a bot configuration. diff --git a/run_csbot.py b/run_csbot.py deleted file mode 100755 index 1aabcd2d..00000000 --- a/run_csbot.py +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env python -from csbot import main -main(auto_envvar_prefix='CSBOT') diff --git a/setup.py b/setup.py index b8598f7b..f0e40936 100644 --- a/setup.py +++ b/setup.py @@ -4,6 +4,18 @@ setup( name='csbot', version='0.1', + author='Alan Briolat', + author_email='alan@briol.at', + url='https://github.com/HackSoc/csbot', packages=['csbot', 'csbot.plugins'], package_dir={'': 'src'}, + classifiers=[ + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + ], + entry_points={ + 'console_scripts': [ + 'csbot = csbot:main', + ], + }, ) diff --git a/src/csbot/__init__.py b/src/csbot/__init__.py index f04522c6..2ab02c29 100644 --- a/src/csbot/__init__.py +++ b/src/csbot/__init__.py @@ -13,7 +13,10 @@ LOG = logging.getLogger(__name__) -@click.command(context_settings={'help_option_names': ['-h', '--help']}) +@click.command(context_settings={ + 'help_option_names': ['-h', '--help'], + 'auto_envvar_prefix': 'CSBOT', +}) @click.option('--debug', '-d', is_flag=True, default=False, help='Turn on debug logging for the bot.') @click.option('--debug-irc', is_flag=True, default=False, From c597818230e7db38d8b0ce849c7cbc96b67fdb6d Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Fri, 7 Jun 2019 16:43:57 +0100 Subject: [PATCH 55/66] Move runtime requirements to setup.py --- pytest.ini | 2 ++ requirements.txt | 12 +----------- setup.cfg | 2 -- setup.py | 14 ++++++++++++++ 4 files changed, 17 insertions(+), 13 deletions(-) delete mode 100644 setup.cfg diff --git a/pytest.ini b/pytest.ini index d813a6be..c31864df 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,5 @@ [pytest] +testpaths = tests/ +addopts = --cov markers = bot: mark a test as Bot-based rather than IRCClient-based diff --git a/requirements.txt b/requirements.txt index 2ed71c53..3936899e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,4 @@ -# Requirements for deployment -click>=6.2,<7.0 -straight.plugin==1.4.0-post-1 -pymongo>=3.6.0 -requests>=2.9.1,<3.0.0 -lxml>=2.3.5 -aiogoogle==0.1.13 -isodate>=0.5.1 -aiohttp>=3.5.1,<4.0 -async_generator>=1.10,<2.0 -rollbar +-e . # Requirements for unit testing pytest==4.2.0 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 39baf331..00000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[pytest] -testpaths = tests/ diff --git a/setup.py b/setup.py index f0e40936..75a8ce32 100644 --- a/setup.py +++ b/setup.py @@ -13,6 +13,20 @@ 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', ], + install_requires=[ + 'click>=6.2,<7.0', + 'straight.plugin==1.4.0-post-1', + 'pymongo>=3.6.0', + 'requests>=2.9.1,<3.0.0', + 'lxml>=2.3.5', + 'aiogoogle>=0.1.13', + 'isodate>=0.5.1', + 'aiohttp>=3.5.1,<4.0', + 'async_generator', + 'rollbar', + ], + extras_require={ + }, entry_points={ 'console_scripts': [ 'csbot = csbot:main', From 343bc540a9eabc02eb4f1161d3f6b064d61b10b8 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Fri, 7 Jun 2019 16:47:53 +0100 Subject: [PATCH 56/66] Use tox in Travis CI --- .travis.yml | 16 +++++++++------- tox.ini | 7 +++++++ 2 files changed, 16 insertions(+), 7 deletions(-) create mode 100644 tox.ini diff --git a/.travis.yml b/.travis.yml index b10f3e1d..7f41352b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,17 @@ sudo: false dist: xenial language: python -python: - - 3.6 - - 3.7 - install: - - pip install -r requirements.txt + - pip install tox + +matrix: + include: + - python: '3.6' + env: TOXENV=py36 + - python: '3.7' + env: TOXENV=py37 -script: - - pytest -v --cov +script: tox after_success: - coveralls diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..4535cf5c --- /dev/null +++ b/tox.ini @@ -0,0 +1,7 @@ +[tox] +envlist = py36,py37 +skipsdist = True + +[testenv] +deps = -r requirements.txt +commands = python -m pytest {posargs} \ No newline at end of file From 97e7eb587eae4e668111e322b8d0ba5a27cc47bf Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Fri, 7 Jun 2019 17:40:22 +0100 Subject: [PATCH 57/66] Fix up docker build for new layout --- .dockerignore | 2 ++ Dockerfile | 6 +++--- docker-entrypoint.sh | 2 +- pytest.ini | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..b6f656eb --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +**/__pycache__ +**/*.pyc \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 393ed32d..2c6c95c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,9 +2,9 @@ FROM python:3.7 VOLUME /app WORKDIR /app -COPY csbot ./csbot -COPY csbot.*.cfg requirements.txt run_csbot.py docker-entrypoint.sh ./ -RUN find . -name '*.pyc' -delete +COPY src ./src +COPY tests ./tests +COPY setup.py requirements.txt pytest.ini docker-entrypoint.sh csbot.*.cfg ./ RUN pip install -r requirements.txt diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index ffec0da1..1ed50f40 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -1,3 +1,3 @@ #!/bin/bash -exec ./run_csbot.py $@ +exec csbot $@ diff --git a/pytest.ini b/pytest.ini index c31864df..a2b41d33 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] testpaths = tests/ -addopts = --cov +addopts = --cov=src/ markers = bot: mark a test as Bot-based rather than IRCClient-based From a82ebd525ed4014217d4b341ca69c17b8a2f88a8 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Fri, 7 Jun 2019 17:52:24 +0100 Subject: [PATCH 58/66] Don't build image twice on docker hub, re-use built image for tests --- docker-compose.test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.test.yml b/docker-compose.test.yml index bfdaa413..7089181a 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -2,6 +2,6 @@ version: "3" services: sut: - build: . + image: "${IMAGE_NAME}" entrypoint: pytest command: [] \ No newline at end of file From 137588c2c507672598b7a56d3743f85f91ecca62 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Fri, 7 Jun 2019 17:55:38 +0100 Subject: [PATCH 59/66] Re-enable coveralls in Travis --- .travis.yml | 1 + requirements.txt | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7f41352b..c14f5be4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,7 @@ dist: xenial language: python install: - pip install tox + - pip install python-coveralls matrix: include: diff --git a/requirements.txt b/requirements.txt index 3936899e..a0769ebf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,11 +6,10 @@ pytest-asyncio==0.10.0 pytest-aiohttp==0.3.0 #aioresponses==0.6.0 git+https://github.com/alanbriolat/aioresponses.git@callback-coroutines#egg=aioresponses -pytest-cov==2.6.1 +pytest-cov asynctest==0.12.2 aiofastforward==0.0.17 responses -python-coveralls>=2.6.0,<2.7.0 mongomock # Requirements for documentation From 88ba430a86d69b7b925f9e128c08b9c5d627dccb Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Fri, 7 Jun 2019 20:33:47 +0100 Subject: [PATCH 60/66] Remove redundant docker-entrypoint.sh --- Dockerfile | 5 ++--- docker-compose.test.yml | 3 +-- docker-compose.yml | 2 +- docker-entrypoint.sh | 3 --- 4 files changed, 4 insertions(+), 9 deletions(-) delete mode 100755 docker-entrypoint.sh diff --git a/Dockerfile b/Dockerfile index 2c6c95c6..d130733f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,12 +4,11 @@ VOLUME /app WORKDIR /app COPY src ./src COPY tests ./tests -COPY setup.py requirements.txt pytest.ini docker-entrypoint.sh csbot.*.cfg ./ +COPY setup.py requirements.txt pytest.ini csbot.*.cfg ./ RUN pip install -r requirements.txt ARG SOURCE_COMMIT ENV SOURCE_COMMIT $SOURCE_COMMIT -ENTRYPOINT ["./docker-entrypoint.sh"] -CMD ["./csbot.cfg"] +CMD ["csbot", "csbot.cfg"] diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 7089181a..0f458c23 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -3,5 +3,4 @@ version: "3" services: sut: image: "${IMAGE_NAME}" - entrypoint: pytest - command: [] \ No newline at end of file + command: pytest \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 964c89dd..cc278b75 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ services: - ./deploy.env environment: MONGODB_URI: mongodb://mongodb:27017/csbot - command: ${CSBOT_CONFIG:-csbot.cfg} + command: csbot ${CSBOT_CONFIG:-csbot.cfg} ports: - "127.0.0.1:8180:80" labels: diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh deleted file mode 100755 index 1ed50f40..00000000 --- a/docker-entrypoint.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -exec csbot $@ From 481701ec557bc959a1b6ec56448a18cdd413c114 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Fri, 7 Jun 2019 21:56:44 +0100 Subject: [PATCH 61/66] Actually fix coveralls this time? Also use coveralls-python instead of python-coveralls. --- .travis.yml | 8 ++------ tox.ini | 10 ++++++++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index c14f5be4..16c31f2e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,20 +3,16 @@ dist: xenial language: python install: - pip install tox - - pip install python-coveralls matrix: include: - python: '3.6' - env: TOXENV=py36 + env: TOXENV=py36-coveralls - python: '3.7' - env: TOXENV=py37 + env: TOXENV=py37-coveralls script: tox -after_success: - - coveralls - cache: directories: - $HOME/.cache/pip diff --git a/tox.ini b/tox.ini index 4535cf5c..789009b1 100644 --- a/tox.ini +++ b/tox.ini @@ -3,5 +3,11 @@ envlist = py36,py37 skipsdist = True [testenv] -deps = -r requirements.txt -commands = python -m pytest {posargs} \ No newline at end of file +passenv = TRAVIS TRAVIS_* +deps = + -r requirements.txt + coveralls: coveralls +commands = + python -m pytest {posargs} + # Try to run coveralls, but don't fail if coveralls fails + coveralls: - coveralls From e356d8134c0a20213bd79c2bef388e5fde89c814 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Sat, 8 Jun 2019 16:35:14 +0100 Subject: [PATCH 62/66] docker: run as non-root user, reduce context with .dockerignore whitelist --- .dockerignore | 12 ++++++++++++ Dockerfile | 14 ++++++++------ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.dockerignore b/.dockerignore index b6f656eb..c4a51c35 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,14 @@ +# Exclude everything by default +* + +# Whitelist specific files +!src/**/* +!tests/**/* +!setup.py +!requirements.txt +!pytest.ini +!csbot.*.cfg + +# Exclude Python temp files **/__pycache__ **/*.pyc \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index d130733f..ed86684f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,16 @@ FROM python:3.7 -VOLUME /app -WORKDIR /app -COPY src ./src -COPY tests ./tests -COPY setup.py requirements.txt pytest.ini csbot.*.cfg ./ +ARG UID=9000 +ARG GID=9000 + +RUN groupadd -g $GID app \ + && useradd -u $UID -g $GID --no-create-home app +COPY --chown=app:app . /app +WORKDIR /app RUN pip install -r requirements.txt ARG SOURCE_COMMIT ENV SOURCE_COMMIT $SOURCE_COMMIT - +USER app:app CMD ["csbot", "csbot.cfg"] From 260a65262ff4dca4ccf9bb57c7aebe755322b51a Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Sat, 8 Jun 2019 16:46:19 +0100 Subject: [PATCH 63/66] Bump version (somewhat arbitrarily), use setuptools not distutils... --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 75a8ce32..02021ce4 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,9 @@ -from distutils.core import setup +import setuptools -setup( +setuptools.setup( name='csbot', - version='0.1', + version='0.3.0', author='Alan Briolat', author_email='alan@briol.at', url='https://github.com/HackSoc/csbot', From d78c71b6027e91952cd04e96825bcda0a898de0d Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Sat, 8 Jun 2019 17:45:56 +0100 Subject: [PATCH 64/66] Add --version option to csbot command --- src/csbot/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/csbot/__init__.py b/src/csbot/__init__.py index 2ab02c29..d47568d0 100644 --- a/src/csbot/__init__.py +++ b/src/csbot/__init__.py @@ -10,6 +10,14 @@ from .core import Bot +__version__ = None +try: + import pkg_resources + __version__ = pkg_resources.get_distribution('csbot').version +except (pkg_resources.DistributionNotFound, ImportError): + pass + + LOG = logging.getLogger(__name__) @@ -17,6 +25,7 @@ 'help_option_names': ['-h', '--help'], 'auto_envvar_prefix': 'CSBOT', }) +@click.version_option(version=__version__) @click.option('--debug', '-d', is_flag=True, default=False, help='Turn on debug logging for the bot.') @click.option('--debug-irc', is_flag=True, default=False, From 9f7c69c6cf8a6a3f0595d4f108720f859b4bd9f8 Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Sat, 8 Jun 2019 19:52:16 +0100 Subject: [PATCH 65/66] Tweaks --- README.rst | 1 - docker-compose.test.yml | 2 +- requirements.txt | 4 ++-- setup.py | 2 -- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index bc2f4bb9..fd77afdc 100644 --- a/README.rst +++ b/README.rst @@ -14,7 +14,6 @@ and running [1]_:: $ python3 -m venv venv3 $ source venv3/bin/activate $ pip install -r requirements.txt - $ pip install -e . $ csbot --help Look at ``csbot.deploy.cfg`` for an example of a bot configuration. diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 0f458c23..503a0377 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -3,4 +3,4 @@ version: "3" services: sut: image: "${IMAGE_NAME}" - command: pytest \ No newline at end of file + command: pytest diff --git a/requirements.txt b/requirements.txt index a0769ebf..b9e09eae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,3 @@ --e . - # Requirements for unit testing pytest==4.2.0 pytest-asyncio==0.10.0 @@ -15,3 +13,5 @@ mongomock # Requirements for documentation # (commented out to save build time) #sphinx + +-e . diff --git a/setup.py b/setup.py index 02021ce4..51b78d8c 100644 --- a/setup.py +++ b/setup.py @@ -25,8 +25,6 @@ 'async_generator', 'rollbar', ], - extras_require={ - }, entry_points={ 'console_scripts': [ 'csbot = csbot:main', From b62c1fd008bfef73709f82a0be943b14018fcafd Mon Sep 17 00:00:00 2001 From: Alan Briolat Date: Sat, 8 Jun 2019 22:08:34 +0100 Subject: [PATCH 66/66] Change webhook port, can't listen on port 80 as non-root user --- csbot.deploy.cfg | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/csbot.deploy.cfg b/csbot.deploy.cfg index 10207210..644dc49b 100644 --- a/csbot.deploy.cfg +++ b/csbot.deploy.cfg @@ -49,7 +49,7 @@ end = [webserver] host = 0.0.0.0 -port = 80 +port = 8000 [github] # Re-usable format strings diff --git a/docker-compose.yml b/docker-compose.yml index cc278b75..626217a1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,7 @@ services: MONGODB_URI: mongodb://mongodb:27017/csbot command: csbot ${CSBOT_CONFIG:-csbot.cfg} ports: - - "127.0.0.1:8180:80" + - "127.0.0.1:8180:8000" labels: - com.centurylinklabs.watchtower.enable=${CSBOT_WATCHTOWER:-false}