From 5fdaf5cc02a1dfb99d65c94f0100a0abd3f3d506 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 5 Feb 2019 11:33:32 +0000 Subject: [PATCH 1/6] port over --- synapse/app/_base.py | 61 ++++++++++++++++++++++++++++++++ synapse/app/appservice.py | 7 +--- synapse/app/client_reader.py | 13 +------ synapse/app/event_creator.py | 13 +------ synapse/app/federation_reader.py | 13 +------ synapse/app/federation_sender.py | 12 +------ synapse/app/frontend_proxy.py | 13 +------ synapse/app/media_repository.py | 13 +------ synapse/app/pusher.py | 3 +- synapse/app/synchrotron.py | 7 +--- synapse/app/user_dir.py | 13 +------ synapse/config/logger.py | 16 +++------ synapse/server.py | 1 + 13 files changed, 77 insertions(+), 108 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 5b97a54d45a1..40c8b2fdbb69 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -15,19 +15,26 @@ import gc import logging +import signal import sys +import traceback import psutil from daemonize import Daemonize from twisted.internet import error, reactor +from twisted.protocols.tls import TLSMemoryBIOFactory from synapse.app import check_bind_error +from synapse.crypto import context_factory from synapse.util import PreserveLoggingContext from synapse.util.rlimit import change_resource_limit logger = logging.getLogger(__name__) +_sighup_callbacks = [] +register_sighup = _sighup_callbacks.append + def start_worker_reactor(appname, config): """ Run the reactor in the main process @@ -189,3 +196,57 @@ def listen_ssl( logger.info("Synapse now listening on port %d (TLS)", port) return r + + +def refresh_certificate(hs): + """ + Refresh the TLS certificates that Synapse is using by re-reading them from + disk and updating the TLS context factories to use them. + """ + logging.info("Loading certificate from disk...") + hs.config.read_certificate_from_disk() + hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config) + hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( + hs.config + ) + logging.info("Certificate loaded.") + + if hs._listening_services: + logging.info("Updating context factories...") + for i in hs._listening_services: + if isinstance(i.factory, TLSMemoryBIOFactory): + i.factory = TLSMemoryBIOFactory( + hs.tls_server_context_factory, + False, + i.factory.wrappedFactory + ) + logging.info("Context factories updated.") + + +def start(hs, listeners=None): + """ + Start a Synapse server or worker. + """ + try: + # Set up the SIGHUP machinery. + if hasattr(signal, "SIGHUP"): + def handle_sighup(*args, **kwargs): + for i in _sighup_callbacks: + i(hs) + + signal.signal(signal.SIGHUP, handle_sighup) + + register_sighup(refresh_certificate) + + # Load the certificate from disk. + refresh_certificate(hs) + + # It is now safe to start your Synapse. + hs.start_listening(listeners) + hs.get_datastore().start_profiling() + except Exception: + traceback.print_exc(file=sys.stderr) + reactor = hs.get_reactor() + if reactor.running: + reactor.stop() + sys.exit(1) diff --git a/synapse/app/appservice.py b/synapse/app/appservice.py index 8559e141af47..33107f56d137 100644 --- a/synapse/app/appservice.py +++ b/synapse/app/appservice.py @@ -168,12 +168,7 @@ def start(config_options): ) ps.setup() - ps.start_listening(config.worker_listeners) - - def start(): - ps.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ps, config.worker_listeners) _base.start_worker_reactor("synapse-appservice", config) diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py index f8a417cb6070..a9d214702286 100644 --- a/synapse/app/client_reader.py +++ b/synapse/app/client_reader.py @@ -25,7 +25,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -173,17 +172,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-client-reader", config) diff --git a/synapse/app/event_creator.py b/synapse/app/event_creator.py index 656e0edc0fbc..b8e51961528b 100644 --- a/synapse/app/event_creator.py +++ b/synapse/app/event_creator.py @@ -25,7 +25,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -194,17 +193,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-event-creator", config) diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index 3de27151328c..42886dbfc149 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -26,7 +26,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.federation.transport.server import TransportLayerServer from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -160,17 +159,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-federation-reader", config) diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py index d944e0517f2e..a461442fdc41 100644 --- a/synapse/app/federation_sender.py +++ b/synapse/app/federation_sender.py @@ -25,7 +25,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.federation import send_queue from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -192,17 +191,8 @@ def start(config_options): ) ss.setup() + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) _base.start_worker_reactor("synapse-federation-sender", config) diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py index d9ef6edc3c6d..d5b954361d9a 100644 --- a/synapse/app/frontend_proxy.py +++ b/synapse/app/frontend_proxy.py @@ -26,7 +26,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.servlet import RestServlet, parse_json_object_from_request from synapse.http.site import SynapseSite @@ -250,17 +249,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-frontend-proxy", config) diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py index 4ecf64031bf7..d4cc4e9443c7 100644 --- a/synapse/app/media_repository.py +++ b/synapse/app/media_repository.py @@ -26,7 +26,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy from synapse.metrics.resource import METRICS_PREFIX, MetricsResource @@ -160,17 +159,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-media-repository", config) diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index 83b0863f00f4..cbf0d67f51fa 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -224,11 +224,10 @@ def start(config_options): ) ps.setup() - ps.start_listening(config.worker_listeners) def start(): + _base.start(ps, config.worker_listeners) ps.get_pusherpool().start() - ps.get_datastore().start_profiling() reactor.callWhenRunning(start) diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py index 0354e82bf850..9163b56d86fb 100644 --- a/synapse/app/synchrotron.py +++ b/synapse/app/synchrotron.py @@ -445,12 +445,7 @@ def start(config_options): ) ss.setup() - ss.start_listening(config.worker_listeners) - - def start(): - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-synchrotron", config) diff --git a/synapse/app/user_dir.py b/synapse/app/user_dir.py index 176d55a78333..d1ab9512cd07 100644 --- a/synapse/app/user_dir.py +++ b/synapse/app/user_dir.py @@ -26,7 +26,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -220,17 +219,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-user-dir", config) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index a795e39b1acc..4b938053fb75 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -15,7 +15,6 @@ import logging import logging.config import os -import signal import sys from string import Template @@ -24,6 +23,7 @@ from twisted.logger import STDLibLogObserver, globalLogBeginner import synapse +from synapse.app import _base as appbase from synapse.util.logcontext import LoggingContextFilter from synapse.util.versionstring import get_version_string @@ -127,7 +127,7 @@ def generate_files(self, config): ) -def setup_logging(config, use_worker_options=False, register_sighup=None): +def setup_logging(config, use_worker_options=False): """ Set up python logging Args: @@ -140,12 +140,6 @@ def setup_logging(config, use_worker_options=False, register_sighup=None): register_sighup (func | None): Function to call to register a sighup handler. """ - if not register_sighup: - if getattr(signal, "SIGHUP"): - register_sighup = lambda x: signal.signal(signal.SIGHUP, x) - else: - register_sighup = lambda x: None - log_config = (config.worker_log_config if use_worker_options else config.log_config) log_file = (config.worker_log_file if use_worker_options @@ -187,7 +181,7 @@ def sighup(signum, stack): else: handler = logging.StreamHandler() - def sighup(signum, stack): + def sighup(*args): pass handler.setFormatter(formatter) @@ -200,14 +194,14 @@ def load_log_config(): with open(log_config, 'r') as f: logging.config.dictConfig(yaml.load(f)) - def sighup(signum, stack): + def sighup(*args): # it might be better to use a file watcher or something for this. load_log_config() logging.info("Reloaded log config from %s due to SIGHUP", log_config) load_log_config() - register_sighup(sighup) + appbase.register_sighup(sighup) # make sure that the first thing we log is a thing we can grep backwards # for diff --git a/synapse/server.py b/synapse/server.py index 6c521016163f..cb35a7bab3cc 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -196,6 +196,7 @@ def __init__(self, hostname, reactor=None, **kwargs): self._reactor = reactor self.hostname = hostname self._building = {} + self._listening_services = [] self.clock = Clock(reactor) self.distributor = Distributor() From 3fa83ab47f8003f29dd15176ed04116645c4e559 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 5 Feb 2019 11:44:47 +0000 Subject: [PATCH 2/6] changelog --- changelog.d/4567.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4567.misc diff --git a/changelog.d/4567.misc b/changelog.d/4567.misc new file mode 100644 index 000000000000..96a2e0aefc16 --- /dev/null +++ b/changelog.d/4567.misc @@ -0,0 +1 @@ +Reduce duplication of ``synapse.app`` code. From 177969689d82716236d338519a8d7c70feca6ac9 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 5 Feb 2019 11:50:12 +0000 Subject: [PATCH 3/6] fix --- synapse/app/homeserver.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 250a17cef865..33a96ae51b6d 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -17,7 +17,6 @@ import gc import logging import os -import signal import sys import traceback @@ -328,20 +327,11 @@ def setup(config_options): # generating config files and shouldn't try to continue. sys.exit(0) - sighup_callbacks = [] synapse.config.logger.setup_logging( config, - use_worker_options=False, - register_sighup=sighup_callbacks.append + use_worker_options=False ) - def handle_sighup(*args, **kwargs): - for i in sighup_callbacks: - i(*args, **kwargs) - - if hasattr(signal, "SIGHUP"): - signal.signal(signal.SIGHUP, handle_sighup) - events.USE_FROZEN_DICTS = config.use_frozen_dicts database_engine = create_engine(config.database_config) @@ -400,8 +390,6 @@ def refresh_certificate(*args): ) logging.info("Context factories updated.") - sighup_callbacks.append(refresh_certificate) - @defer.inlineCallbacks def start(): try: From 235d19f44e432154c04e85b0cfd97ebdae7390c8 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 8 Feb 2019 14:43:09 +0000 Subject: [PATCH 4/6] fix --- synapse/app/_base.py | 11 ++++++++--- synapse/app/homeserver.py | 5 +++-- synapse/config/tls.py | 5 +---- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 40c8b2fdbb69..ff02081b8d89 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -203,12 +203,17 @@ def refresh_certificate(hs): Refresh the TLS certificates that Synapse is using by re-reading them from disk and updating the TLS context factories to use them. """ - logging.info("Loading certificate from disk...") - hs.config.read_certificate_from_disk() - hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config) hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( hs.config ) + + if hs.config.no_tls == True: + logging.info("Serving TLS is disabled, not lpa") + return + + logging.info("Loading certificate from disk...") + hs.config.read_certificate_from_disk() + hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config) logging.info("Certificate loaded.") if hs._listening_services: diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 33a96ae51b6d..0a43e69c891d 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -94,9 +94,10 @@ def _listener_http(self, config, listener_config): site_tag = listener_config.get("tag", port) if tls and config.no_tls: - raise ConfigError( - "Listener on port %i has TLS enabled, but no_tls is set" % (port,), + logger.info( + "Listener on port %i has TLS enabled, but no_tls is set, skipping" % (port,) ) + return resources = {} for res in listener_config["resources"]: diff --git a/synapse/config/tls.py b/synapse/config/tls.py index b5f2cfd9b7cc..ccf500275542 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -108,10 +108,7 @@ def read_certificate_from_disk(self): ) ) - if not self.no_tls: - self.tls_private_key = self.read_tls_private_key(self.tls_private_key_file) - - self.tls_fingerprints = list(self._original_tls_fingerprints) + self.tls_private_key = self.read_tls_private_key(self.tls_private_key_file) # Check that our own certificate is included in the list of fingerprints # and include it if it is not. From 322e47267ff715798211c568e48ac33805484c6a Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 11 Feb 2019 20:08:15 +1100 Subject: [PATCH 5/6] fix typo --- synapse/app/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index fb7a9198bcc5..82bea617c659 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -218,7 +218,7 @@ def refresh_certificate(hs): ) if hs.config.no_tls == True: - logging.info("Serving TLS is disabled, not lpa") + logging.info("Serving TLS is disabled, not loading certificates.") return logging.info("Loading certificate from disk...") From fe99db298dfe6dbb9342ac8b091580db51cdcaf7 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 11 Feb 2019 11:48:09 +0000 Subject: [PATCH 6/6] fix lint --- synapse/app/_base.py | 2 +- synapse/app/homeserver.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 82e7802e48c0..79da72db60ad 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -217,7 +217,7 @@ def refresh_certificate(hs): hs.config ) - if hs.config.no_tls == True: + if hs.config.no_tls: logging.info("Serving TLS is disabled, not loading certificates.") return diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index a6e8916e2dc1..a67937d09c11 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -92,7 +92,8 @@ def _listener_http(self, config, listener_config): if tls and config.no_tls: logger.info( - "Listener on port %i has TLS enabled, but no_tls is set, skipping" % (port,) + "Listener on port %i has TLS enabled, but no_tls is set, skipping", + port, ) return