From c833250182fe3dffd1b68433b2142cb242610b57 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 13 Aug 2020 13:11:02 +0300 Subject: [PATCH 1/9] Add endpoint for sending forwards --- synapse/rest/__init__.py | 2 + synapse/rest/client/v2_alpha/forward_event.py | 118 ++++++++++++++++++ synapse/rest/client/versions.py | 2 + 3 files changed, 122 insertions(+) create mode 100644 synapse/rest/client/v2_alpha/forward_event.py diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 46e458e95ba0..811982578ff9 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -47,6 +47,7 @@ register, relations, report_event, + forward_event, room_keys, room_upgrade_rest_servlet, sendtodevice, @@ -108,6 +109,7 @@ def register_servlets(client_resource, hs): tags.register_servlets(hs, client_resource) account_data.register_servlets(hs, client_resource) report_event.register_servlets(hs, client_resource) + forward_event.register_servlets(hs, client_resource) openid.register_servlets(hs, client_resource) notifications.register_servlets(hs, client_resource) devices.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/forward_event.py b/synapse/rest/client/v2_alpha/forward_event.py new file mode 100644 index 000000000000..d77c412f19e6 --- /dev/null +++ b/synapse/rest/client/v2_alpha/forward_event.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Tulir Asokan +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import logging + +from synapse.api.errors import Codes, SynapseError, AuthError +from synapse.http.servlet import RestServlet, parse_integer +from synapse.logging.opentracing import set_tag + +from ._base import client_patterns + +logger = logging.getLogger(__name__) + + +class RoomEventForwardServlet(RestServlet): + """ + PUT /net.maunium.msc2730/rooms/{room_id}/event/{event_id}/forward/{target_room_id}/{txn_id} + """ + + PATTERNS = client_patterns( + ("/net.maunium.msc2730/rooms/(?P[^/]*)/event/(?P[^/]*)" + "/forward/(?P[^/]*)/(?P.*)"), + releases=(), # This is an unstable feature + ) + + _data_key = "net.maunium.msc2730.forwarded" + _err_not_forwardable = "NET.MAUNIUM.MSC2730_NOT_FORWARDABLE" + + def __init__(self, hs): + super().__init__() + self.event_creation_handler = hs.get_event_creation_handler() + self.event_handler = hs.get_event_handler() + self.store = hs.get_datastore() + self.auth = hs.get_auth() + + async def on_PUT(self, request, room_id, event_id, target_room_id, txn_id): + requester = await self.auth.get_user_by_req(request, allow_guest=True) + + try: + event = await self.event_handler.get_event( + requester.user, room_id, event_id + ) + except AuthError: + event = None + if not event: + raise SynapseError(404, "Event not found.", errcode=Codes.NOT_FOUND) + + if event.is_state(): + raise SynapseError(401, "State events cannot be forwarded.", + errcode=self._err_not_forwardable) + elif event.redacts: + raise SynapseError(401, "Redaction events cannot be forwarded.", + errcode=self._err_not_forwardable) + elif event.internal_metadata.is_redacted(): + raise SynapseError(401, "Redacted events cannot be forwarded.", + errcode=self._err_not_forwardable) + + event_id = event.event_id + event_dict = event.get_dict() + + content = event_dict.pop("content") + unsigned = event_dict.pop("unsigned", {}) + event_type = event_dict.pop("type") + has_forward_meta = self._data_key in content + try: + is_valid_forward = has_forward_meta and unsigned[self._data_key]["valid"] + except (KeyError, TypeError): + is_valid_forward = False + + if has_forward_meta and not is_valid_forward: + raise SynapseError(401, "Event contains invalid forward metadata.", + errcode=self._err_not_forwardable) + elif not has_forward_meta: + content[self._data_key] = event_dict + content[self._data_key]["unsigned"] = { + "room_version": await self.store.get_room_version(event.room_id), + # TODO add sender profile info here + } + + forwarded_event_dict = { + "type": event_type, + "content": content, + "room_id": target_room_id, + "sender": requester.user.to_string(), + "unsigned": { + self._data_key: { + "valid": True, + "event_id": event_id, + } + } + } + + if b"ts" in request.args and requester.app_service: + forwarded_event_dict["origin_server_ts"] = parse_integer(request, "ts", 0) + + forwarded_event, _ = await self.event_creation_handler.create_and_send_nonmember_event( + requester, forwarded_event_dict, txn_id=txn_id + ) + + set_tag("event_id", forwarded_event.event_id) + return 200, {"event_id": forwarded_event.event_id} + + +def register_servlets(hs, http_server): + RoomEventForwardServlet(hs).register(http_server) diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 0d668df0b6f4..d58664efbf1d 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -60,6 +60,8 @@ def on_GET(self, request): "org.matrix.e2e_cross_signing": True, # Implements additional endpoints as described in MSC2432 "org.matrix.msc2432": True, + # Implements endpoint and forwarded event validation as described in MSC2730 + "net.maunium.msc2730": True, }, }, ) From 1357a7559216c4568dc931161b59e69c20d8ab5b Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 13 Aug 2020 13:24:02 +0300 Subject: [PATCH 2/9] Validate forwards in events received over federation --- synapse/handlers/federation.py | 39 +++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 593932adb788..c14ddac69364 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -50,6 +50,7 @@ ) from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion, RoomVersions from synapse.crypto.event_signing import compute_event_signature +from synapse.federation.federation_base import FederationBase, event_from_pdu_json from synapse.event_auth import auth_types_for_event from synapse.events import EventBase from synapse.events.snapshot import EventContext @@ -99,7 +100,7 @@ class _NewEventInfo: auth_events = attr.ib(type=Optional[StateMap[EventBase]], default=None) -class FederationHandler(BaseHandler): +class FederationHandler(BaseHandler, FederationBase): """Handles events that originated from federation. Responsible for: a) handling received Pdus before handing them on as Events to the rest @@ -677,6 +678,35 @@ async def _get_events_from_store_or_dest( return fetched_events + _forwarded_key = "net.maunium.msc2730.forwarded" + + async def _validate_forwarded_event(self, event: EventBase) -> Tuple[bool, Optional[str]]: + try: + source_evt_dict = {**event.content[self._forwarded_key]} + room_version = source_evt_dict["unsigned"]["room_version"] + source_evt_dict["type"] = event.type + source_evt_dict["content"] = {**event.content} + del source_evt_dict["unsigned"] + del source_evt_dict["content"][self._forwarded_key] + except (KeyError, TypeError): + logger.exception("Failed to read forward data") + return False, None + try: + source_evt = event_from_pdu_json(source_evt_dict, room_version) + except SynapseError: + logger.exception("Failed to parse forward data") + return False, None + try: + checked_evt = await self._check_sigs_and_hash(room_version, source_evt) + # _check_sigs_and_hash returns a redacted event if hash validation failed and + # a SynapseError if signature validation failed. In both those cases, we want to + # mark the forward as invalid. + valid = not checked_evt.internal_metadata.is_redacted() + except SynapseError: + logger.exception("Failed to validate forward signature") + valid = False + return valid, source_evt.event_id + async def _process_received_pdu( self, origin: str, event: EventBase, state: Optional[Iterable[EventBase]], ): @@ -697,6 +727,13 @@ async def _process_received_pdu( logger.debug("[%s %s] Processing event: %s", room_id, event_id, event) + if not event.is_state() and not event.redacts and self._forwarded_key in event.content: + valid, event_id = await self._validate_forwarded_event(event) + event.unsigned[self._forwarded_key] = { + "valid": valid, + "event_id": event_id, + } + try: context = await self._handle_new_event(origin, event, state=state) except AuthError as e: From ce02e755c1b06a4f7f61df729d05fd15f41124bf Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 13 Aug 2020 13:31:45 +0300 Subject: [PATCH 3/9] Fix handling room versions --- synapse/handlers/federation.py | 9 ++++++++- synapse/rest/client/v2_alpha/forward_event.py | 3 ++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index c14ddac69364..79b6ccf665c7 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -683,7 +683,7 @@ async def _get_events_from_store_or_dest( async def _validate_forwarded_event(self, event: EventBase) -> Tuple[bool, Optional[str]]: try: source_evt_dict = {**event.content[self._forwarded_key]} - room_version = source_evt_dict["unsigned"]["room_version"] + room_version_identifier = source_evt_dict["unsigned"]["room_version"] source_evt_dict["type"] = event.type source_evt_dict["content"] = {**event.content} del source_evt_dict["unsigned"] @@ -691,11 +691,18 @@ async def _validate_forwarded_event(self, event: EventBase) -> Tuple[bool, Optio except (KeyError, TypeError): logger.exception("Failed to read forward data") return False, None + + room_version = KNOWN_ROOM_VERSIONS.get(room_version_identifier) + if not room_version: + # We don't support the source room version, so we can't verify the event :( + return False, None + try: source_evt = event_from_pdu_json(source_evt_dict, room_version) except SynapseError: logger.exception("Failed to parse forward data") return False, None + try: checked_evt = await self._check_sigs_and_hash(room_version, source_evt) # _check_sigs_and_hash returns a redacted event if hash validation failed and diff --git a/synapse/rest/client/v2_alpha/forward_event.py b/synapse/rest/client/v2_alpha/forward_event.py index d77c412f19e6..8fbae6597e14 100644 --- a/synapse/rest/client/v2_alpha/forward_event.py +++ b/synapse/rest/client/v2_alpha/forward_event.py @@ -85,8 +85,9 @@ async def on_PUT(self, request, room_id, event_id, target_room_id, txn_id): errcode=self._err_not_forwardable) elif not has_forward_meta: content[self._data_key] = event_dict + room_version = await self.store.get_room_version(event.room_id) content[self._data_key]["unsigned"] = { - "room_version": await self.store.get_room_version(event.room_id), + "room_version": room_version.identifier, # TODO add sender profile info here } From a77a254b31712ee02da6053e1ac04c147fa50b9c Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 13 Aug 2020 13:49:47 +0300 Subject: [PATCH 4/9] Run lint fix --- synapse/handlers/federation.py | 12 ++++-- synapse/rest/__init__.py | 2 +- synapse/rest/client/v2_alpha/forward_event.py | 43 +++++++++++++------ 3 files changed, 40 insertions(+), 17 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 79b6ccf665c7..f52204b1513c 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -50,11 +50,11 @@ ) from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion, RoomVersions from synapse.crypto.event_signing import compute_event_signature -from synapse.federation.federation_base import FederationBase, event_from_pdu_json from synapse.event_auth import auth_types_for_event from synapse.events import EventBase from synapse.events.snapshot import EventContext from synapse.events.validator import EventValidator +from synapse.federation.federation_base import FederationBase, event_from_pdu_json from synapse.handlers._base import BaseHandler from synapse.logging.context import ( make_deferred_yieldable, @@ -680,7 +680,9 @@ async def _get_events_from_store_or_dest( _forwarded_key = "net.maunium.msc2730.forwarded" - async def _validate_forwarded_event(self, event: EventBase) -> Tuple[bool, Optional[str]]: + async def _validate_forwarded_event( + self, event: EventBase + ) -> Tuple[bool, Optional[str]]: try: source_evt_dict = {**event.content[self._forwarded_key]} room_version_identifier = source_evt_dict["unsigned"]["room_version"] @@ -734,7 +736,11 @@ async def _process_received_pdu( logger.debug("[%s %s] Processing event: %s", room_id, event_id, event) - if not event.is_state() and not event.redacts and self._forwarded_key in event.content: + if ( + not event.is_state() + and not event.redacts + and self._forwarded_key in event.content + ): valid, event_id = await self._validate_forwarded_event(event) event.unsigned[self._forwarded_key] = { "valid": valid, diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 811982578ff9..684e94a829f4 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -37,6 +37,7 @@ capabilities, devices, filter, + forward_event, groups, keys, notifications, @@ -47,7 +48,6 @@ register, relations, report_event, - forward_event, room_keys, room_upgrade_rest_servlet, sendtodevice, diff --git a/synapse/rest/client/v2_alpha/forward_event.py b/synapse/rest/client/v2_alpha/forward_event.py index 8fbae6597e14..863d1baa6e75 100644 --- a/synapse/rest/client/v2_alpha/forward_event.py +++ b/synapse/rest/client/v2_alpha/forward_event.py @@ -16,7 +16,7 @@ import logging -from synapse.api.errors import Codes, SynapseError, AuthError +from synapse.api.errors import AuthError, Codes, SynapseError from synapse.http.servlet import RestServlet, parse_integer from synapse.logging.opentracing import set_tag @@ -31,8 +31,10 @@ class RoomEventForwardServlet(RestServlet): """ PATTERNS = client_patterns( - ("/net.maunium.msc2730/rooms/(?P[^/]*)/event/(?P[^/]*)" - "/forward/(?P[^/]*)/(?P.*)"), + ( + "/net.maunium.msc2730/rooms/(?P[^/]*)/event/(?P[^/]*)" + "/forward/(?P[^/]*)/(?P.*)" + ), releases=(), # This is an unstable feature ) @@ -59,14 +61,23 @@ async def on_PUT(self, request, room_id, event_id, target_room_id, txn_id): raise SynapseError(404, "Event not found.", errcode=Codes.NOT_FOUND) if event.is_state(): - raise SynapseError(401, "State events cannot be forwarded.", - errcode=self._err_not_forwardable) + raise SynapseError( + 401, + "State events cannot be forwarded.", + errcode=self._err_not_forwardable, + ) elif event.redacts: - raise SynapseError(401, "Redaction events cannot be forwarded.", - errcode=self._err_not_forwardable) + raise SynapseError( + 401, + "Redaction events cannot be forwarded.", + errcode=self._err_not_forwardable, + ) elif event.internal_metadata.is_redacted(): - raise SynapseError(401, "Redacted events cannot be forwarded.", - errcode=self._err_not_forwardable) + raise SynapseError( + 401, + "Redacted events cannot be forwarded.", + errcode=self._err_not_forwardable, + ) event_id = event.event_id event_dict = event.get_dict() @@ -81,8 +92,11 @@ async def on_PUT(self, request, room_id, event_id, target_room_id, txn_id): is_valid_forward = False if has_forward_meta and not is_valid_forward: - raise SynapseError(401, "Event contains invalid forward metadata.", - errcode=self._err_not_forwardable) + raise SynapseError( + 401, + "Event contains invalid forward metadata.", + errcode=self._err_not_forwardable, + ) elif not has_forward_meta: content[self._data_key] = event_dict room_version = await self.store.get_room_version(event.room_id) @@ -101,13 +115,16 @@ async def on_PUT(self, request, room_id, event_id, target_room_id, txn_id): "valid": True, "event_id": event_id, } - } + }, } if b"ts" in request.args and requester.app_service: forwarded_event_dict["origin_server_ts"] = parse_integer(request, "ts", 0) - forwarded_event, _ = await self.event_creation_handler.create_and_send_nonmember_event( + ( + forwarded_event, + _, + ) = await self.event_creation_handler.create_and_send_nonmember_event( requester, forwarded_event_dict, txn_id=txn_id ) From 3b4d239ab4360643b93489acd005448a5a5fed46 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 13 Aug 2020 13:52:02 +0300 Subject: [PATCH 5/9] Add changelog --- changelog.d/8078.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/8078.feature diff --git a/changelog.d/8078.feature b/changelog.d/8078.feature new file mode 100644 index 000000000000..4fd4f4a4ec2e --- /dev/null +++ b/changelog.d/8078.feature @@ -0,0 +1 @@ +Implement [MSC2730](https://github.com/matrix-org/matrix-doc/pull/2730): Verifiable forwarded events. From 9abb6b1d610091064d40c082961b524d5adc5802 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 13 Aug 2020 14:04:27 +0300 Subject: [PATCH 6/9] Fix things CI complained about --- synapse/handlers/federation.py | 4 ++-- synapse/rest/client/v2_alpha/forward_event.py | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index f52204b1513c..1f5a84943bce 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -741,10 +741,10 @@ async def _process_received_pdu( and not event.redacts and self._forwarded_key in event.content ): - valid, event_id = await self._validate_forwarded_event(event) + valid, forwarded_event_id = await self._validate_forwarded_event(event) event.unsigned[self._forwarded_key] = { "valid": valid, - "event_id": event_id, + "event_id": forwarded_event_id, } try: diff --git a/synapse/rest/client/v2_alpha/forward_event.py b/synapse/rest/client/v2_alpha/forward_event.py index 863d1baa6e75..6612ae98cdf6 100644 --- a/synapse/rest/client/v2_alpha/forward_event.py +++ b/synapse/rest/client/v2_alpha/forward_event.py @@ -110,12 +110,7 @@ async def on_PUT(self, request, room_id, event_id, target_room_id, txn_id): "content": content, "room_id": target_room_id, "sender": requester.user.to_string(), - "unsigned": { - self._data_key: { - "valid": True, - "event_id": event_id, - } - }, + "unsigned": {self._data_key: {"valid": True, "event_id": event_id}}, } if b"ts" in request.args and requester.app_service: From ad7b97c750cd1e59496e0938d3f804c60272c0bd Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 13 Aug 2020 14:06:41 +0300 Subject: [PATCH 7/9] Use more sensible variable name --- synapse/rest/client/v2_alpha/forward_event.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/rest/client/v2_alpha/forward_event.py b/synapse/rest/client/v2_alpha/forward_event.py index 6612ae98cdf6..ef534ac913ab 100644 --- a/synapse/rest/client/v2_alpha/forward_event.py +++ b/synapse/rest/client/v2_alpha/forward_event.py @@ -105,7 +105,7 @@ async def on_PUT(self, request, room_id, event_id, target_room_id, txn_id): # TODO add sender profile info here } - forwarded_event_dict = { + forward_event_dict = { "type": event_type, "content": content, "room_id": target_room_id, @@ -114,13 +114,13 @@ async def on_PUT(self, request, room_id, event_id, target_room_id, txn_id): } if b"ts" in request.args and requester.app_service: - forwarded_event_dict["origin_server_ts"] = parse_integer(request, "ts", 0) + forward_event_dict["origin_server_ts"] = parse_integer(request, "ts", 0) ( forwarded_event, _, ) = await self.event_creation_handler.create_and_send_nonmember_event( - requester, forwarded_event_dict, txn_id=txn_id + requester, forward_event_dict, txn_id=txn_id ) set_tag("event_id", forwarded_event.event_id) From 1e442a4babde3ced700e80de02c022cbf4c362fd Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 13 Aug 2020 16:03:00 +0300 Subject: [PATCH 8/9] Remove forward validation debug logs --- synapse/handlers/federation.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 1f5a84943bce..32355dc34e27 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -691,7 +691,6 @@ async def _validate_forwarded_event( del source_evt_dict["unsigned"] del source_evt_dict["content"][self._forwarded_key] except (KeyError, TypeError): - logger.exception("Failed to read forward data") return False, None room_version = KNOWN_ROOM_VERSIONS.get(room_version_identifier) @@ -702,7 +701,6 @@ async def _validate_forwarded_event( try: source_evt = event_from_pdu_json(source_evt_dict, room_version) except SynapseError: - logger.exception("Failed to parse forward data") return False, None try: @@ -712,7 +710,6 @@ async def _validate_forwarded_event( # mark the forward as invalid. valid = not checked_evt.internal_metadata.is_redacted() except SynapseError: - logger.exception("Failed to validate forward signature") valid = False return valid, source_evt.event_id From f3fa81a3790a3ae9e9ca31564a8929e01d031862 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Fri, 14 Aug 2020 19:07:39 +0300 Subject: [PATCH 9/9] Fix local unsigned event ID when forwarding forwarded event --- synapse/rest/client/v2_alpha/forward_event.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/synapse/rest/client/v2_alpha/forward_event.py b/synapse/rest/client/v2_alpha/forward_event.py index ef534ac913ab..e488633e914f 100644 --- a/synapse/rest/client/v2_alpha/forward_event.py +++ b/synapse/rest/client/v2_alpha/forward_event.py @@ -91,12 +91,15 @@ async def on_PUT(self, request, room_id, event_id, target_room_id, txn_id): except (KeyError, TypeError): is_valid_forward = False - if has_forward_meta and not is_valid_forward: - raise SynapseError( - 401, - "Event contains invalid forward metadata.", - errcode=self._err_not_forwardable, - ) + if has_forward_meta: + if not is_valid_forward: + raise SynapseError( + 401, + "Event contains invalid forward metadata.", + errcode=self._err_not_forwardable, + ) + # Pass through the old event ID to the new unsigned data + event_id = unsigned[self._data_key]["event_id"] elif not has_forward_meta: content[self._data_key] = event_dict room_version = await self.store.get_room_version(event.room_id)