From 40d2a8d2a96f89e922a436336f068573d6472ddb Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Wed, 6 Jan 2021 23:56:44 -0500 Subject: [PATCH 1/6] Add a new dockerfile for worker-mode synapse --- dockerfiles/SynapseWorkers.Dockerfile | 64 +++++++++++++++++++ dockerfiles/synapse/caddy.complement.json | 76 +++++++++++++++++++++++ dockerfiles/synapse/workers-shared.yaml | 59 ++++++++++++++++++ 3 files changed, 199 insertions(+) create mode 100644 dockerfiles/SynapseWorkers.Dockerfile create mode 100644 dockerfiles/synapse/caddy.complement.json create mode 100644 dockerfiles/synapse/workers-shared.yaml diff --git a/dockerfiles/SynapseWorkers.Dockerfile b/dockerfiles/SynapseWorkers.Dockerfile new file mode 100644 index 00000000..05fdf200 --- /dev/null +++ b/dockerfiles/SynapseWorkers.Dockerfile @@ -0,0 +1,64 @@ +# This dockerfile builds on top of Dockerfile-worker and includes a built-in postgres instance +# as well as sets up the homeserver so that it is ready for testing via Complement +FROM matrixdotorg/synapse:workers + +# Tell Complement that we are using its custom CA +ENV COMPLEMENT_CA=true + +# Install postgresql +RUN apt-get update +RUN apt-get install -y postgresql + +# Configure a user and create a database for Synapse +RUN pg_ctlcluster 11 main start && su postgres -c "echo \ + \"ALTER USER postgres PASSWORD 'somesecret'; \ + CREATE DATABASE synapse \ + ENCODING 'UTF8' \ + LC_COLLATE='C' \ + LC_CTYPE='C' \ + template=template0;\" | psql" && pg_ctlcluster 11 main stop + +# Modify the shared homeserver config with postgres support, certificate setup +# and the disabling of rate-limiting +COPY synapse/workers-shared.yaml /conf/workers/shared.yaml + +# Set up TLS certificates using the custom CA +COPY keys/* /ca/ + +# SSL key for the server (can't make the cert until we know the server name) +RUN openssl genrsa -out /conf/server.tls.key 2048 + +# Generate a signing key +RUN generate_signing_key.py -o /conf/server.signing.key + +WORKDIR /root + +# Download a caddy server to stand in front of nginx and terminate TLS using Complement's +# custom CA +RUN curl -OL "https://github.com/caddyserver/caddy/releases/download/v2.3.0/caddy_2.3.0_linux_amd64.tar.gz" && \ + tar xzf caddy_2.3.0_linux_amd64.tar.gz && rm caddy_2.3.0_linux_amd64.tar.gz + +# Copy the caddy config +COPY synapse/caddy.complement.json /root/caddy.json + +# Expose caddy's listener ports +EXPOSE 8008 8448 + +ENTRYPOINT \ + # Replace the server name in the caddy config + sed -i "s/{{ server_name }}/${SERVER_NAME}/g" /root/caddy.json && \ + # Start postgres + pg_ctlcluster 11 main start > /dev/null 2>&1 && \ + # Start caddy + /root/caddy start --config /root/caddy.json > /dev/null 2>&1 && \ + # Set the server name of the homeserver + SYNAPSE_SERVER_NAME=${SERVER_NAME} \ + # No need to report stats here + SYNAPSE_REPORT_STATS=no \ + # Set postgres authentication details which will be placed in the homeserver config file + POSTGRES_PASSWORD=somesecret POSTGRES_USER=postgres POSTGRES_HOST=localhost \ + # Use all available workers + SYNAPSE_WORKERS=* \ + # The script that write the necessary config files and starts supervisord, which in turn + # starts everything else + /configure_workers_and_start.py diff --git a/dockerfiles/synapse/caddy.complement.json b/dockerfiles/synapse/caddy.complement.json new file mode 100644 index 00000000..3373e809 --- /dev/null +++ b/dockerfiles/synapse/caddy.complement.json @@ -0,0 +1,76 @@ +{ + "apps": { + "http": { + "servers": { + "srv0": { + "listen": [ + ":8448" + ], + "routes": [ + { + "match": [ + { + "host": [ + "{{ server_name }}" + ] + } + ], + "handle": [ + { + "handler": "subroute", + "routes": [ + { + "handle": [ + { + "handler": "reverse_proxy", + "upstreams": [ + { + "dial": "localhost:8080" + } + ] + } + ] + } + ] + } + ], + "terminal": true + } + ] + } + } + }, + "tls": { + "automation": { + "policies": [ + { + "subjects": [ + "{{ server_name }}" + ], + "issuers": [ + { + "module": "internal" + } + ], + "on_demand": true + } + ] + } + }, + "pki": { + "certificate_authorities": { + "local": { + "name": "Complement CA", + "root": { + "certificate": "/ca/ca.crt", + "private_key": "/ca/ca.key" + }, + "intermediate": { + "certificate": "/ca/ca.crt", + "private_key": "/ca/ca.key" + } + } + } + } + } + } diff --git a/dockerfiles/synapse/workers-shared.yaml b/dockerfiles/synapse/workers-shared.yaml new file mode 100644 index 00000000..e0572d86 --- /dev/null +++ b/dockerfiles/synapse/workers-shared.yaml @@ -0,0 +1,59 @@ +## Server ## +report_stats: False +trusted_key_servers: [] +enable_registration: true + +## Federation ## + +# disable verification of federation certificates +# +# TODO: Figure out why this is still needed even though we are making use of the custom CA +federation_verify_certificates: false + +# trust certs signed by Complement's CA +federation_custom_ca_list: +- /ca/ca.crt + +# unblacklist RFC1918 addresses +federation_ip_range_blacklist: [] + +# Disable server rate-limiting +rc_federation: + window_size: 1000 + sleep_limit: 10 + sleep_delay: 500 + reject_limit: 99999 + concurrent: 3 + +rc_message: + per_second: 9999 + burst_count: 9999 + +rc_registration: + per_second: 9999 + burst_count: 9999 + +rc_login: + address: + per_second: 9999 + burst_count: 9999 + account: + per_second: 9999 + burst_count: 9999 + failed_attempts: + per_second: 9999 + burst_count: 9999 + +rc_admin_redaction: + per_second: 9999 + burst_count: 9999 + +rc_joins: + local: + per_second: 9999 + burst_count: 9999 + remote: + per_second: 9999 + burst_count: 9999 + +federation_rr_transactions_per_room_per_second: 9999 From c34163cba2b3532eabbfc443ac27d3f79672ace1 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 19 Jan 2021 18:28:02 +0000 Subject: [PATCH 2/6] Download caddy initially; use Complement's CA --- dockerfiles/SynapseWorkers.Dockerfile | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/dockerfiles/SynapseWorkers.Dockerfile b/dockerfiles/SynapseWorkers.Dockerfile index 05fdf200..41866c02 100644 --- a/dockerfiles/SynapseWorkers.Dockerfile +++ b/dockerfiles/SynapseWorkers.Dockerfile @@ -5,6 +5,12 @@ FROM matrixdotorg/synapse:workers # Tell Complement that we are using its custom CA ENV COMPLEMENT_CA=true +# Download a caddy server to stand in front of nginx and terminate TLS using Complement's +# custom CA. +# We include this near the top of the file in order to cache the result. +RUN curl -OL "https://github.com/caddyserver/caddy/releases/download/v2.3.0/caddy_2.3.0_linux_amd64.tar.gz" && \ + tar xzf caddy_2.3.0_linux_amd64.tar.gz && rm caddy_2.3.0_linux_amd64.tar.gz && mv caddy /root + # Install postgresql RUN apt-get update RUN apt-get install -y postgresql @@ -22,22 +28,11 @@ RUN pg_ctlcluster 11 main start && su postgres -c "echo \ # and the disabling of rate-limiting COPY synapse/workers-shared.yaml /conf/workers/shared.yaml -# Set up TLS certificates using the custom CA -COPY keys/* /ca/ - -# SSL key for the server (can't make the cert until we know the server name) -RUN openssl genrsa -out /conf/server.tls.key 2048 - # Generate a signing key RUN generate_signing_key.py -o /conf/server.signing.key WORKDIR /root -# Download a caddy server to stand in front of nginx and terminate TLS using Complement's -# custom CA -RUN curl -OL "https://github.com/caddyserver/caddy/releases/download/v2.3.0/caddy_2.3.0_linux_amd64.tar.gz" && \ - tar xzf caddy_2.3.0_linux_amd64.tar.gz && rm caddy_2.3.0_linux_amd64.tar.gz - # Copy the caddy config COPY synapse/caddy.complement.json /root/caddy.json From 308eeb665b16b5919aefa410e8f1c0c91bf3a389 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 21 Jan 2021 14:39:26 +0000 Subject: [PATCH 3/6] Disable the federation_sender worker for now --- dockerfiles/SynapseWorkers.Dockerfile | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/dockerfiles/SynapseWorkers.Dockerfile b/dockerfiles/SynapseWorkers.Dockerfile index 41866c02..9eda7a19 100644 --- a/dockerfiles/SynapseWorkers.Dockerfile +++ b/dockerfiles/SynapseWorkers.Dockerfile @@ -3,6 +3,7 @@ FROM matrixdotorg/synapse:workers # Tell Complement that we are using its custom CA +# TODO: This doesn't seem to actually enable COMPLEMENT_CA... ENV COMPLEMENT_CA=true # Download a caddy server to stand in front of nginx and terminate TLS using Complement's @@ -52,8 +53,12 @@ ENTRYPOINT \ SYNAPSE_REPORT_STATS=no \ # Set postgres authentication details which will be placed in the homeserver config file POSTGRES_PASSWORD=somesecret POSTGRES_USER=postgres POSTGRES_HOST=localhost \ - # Use all available workers - SYNAPSE_WORKERS=* \ - # The script that write the necessary config files and starts supervisord, which in turn + # Note: This list currently includes all worker types other than federation_sender, as + # Synapse fails to send federation transactions with it enabled. + # https://github.com/matrix-org/synapse/issues/9192 + SYNAPSE_WORKERS=pusher,user_dir,media_repository,appservice,synchrotron,federation_reader,federation_inbound \ + # To use all available workers: + #SYNAPSE_WORKERS=* \ + # Run the script that writes the necessary config files and starts supervisord, which in turn # starts everything else /configure_workers_and_start.py From 88924f6ed961efb0fdef48a2209d3f81ecde9db8 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Thu, 21 Jan 2021 14:41:56 +0000 Subject: [PATCH 4/6] Setting COMPLEMENT_CA as a dockerfile env var is not expected to work --- dockerfiles/SynapseWorkers.Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dockerfiles/SynapseWorkers.Dockerfile b/dockerfiles/SynapseWorkers.Dockerfile index 9eda7a19..44a34802 100644 --- a/dockerfiles/SynapseWorkers.Dockerfile +++ b/dockerfiles/SynapseWorkers.Dockerfile @@ -2,10 +2,6 @@ # as well as sets up the homeserver so that it is ready for testing via Complement FROM matrixdotorg/synapse:workers -# Tell Complement that we are using its custom CA -# TODO: This doesn't seem to actually enable COMPLEMENT_CA... -ENV COMPLEMENT_CA=true - # Download a caddy server to stand in front of nginx and terminate TLS using Complement's # custom CA. # We include this near the top of the file in order to cache the result. From fbc581d88b29c4e850e47af7c69325856d1dac37 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Sat, 23 Jan 2021 01:48:58 +0000 Subject: [PATCH 5/6] No need to generate a signing key in SynapseWorkers.Dockerfile The signing key is already generated by the dockerfile we're inheriting from. --- dockerfiles/SynapseWorkers.Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/dockerfiles/SynapseWorkers.Dockerfile b/dockerfiles/SynapseWorkers.Dockerfile index 44a34802..bed8ceef 100644 --- a/dockerfiles/SynapseWorkers.Dockerfile +++ b/dockerfiles/SynapseWorkers.Dockerfile @@ -25,9 +25,6 @@ RUN pg_ctlcluster 11 main start && su postgres -c "echo \ # and the disabling of rate-limiting COPY synapse/workers-shared.yaml /conf/workers/shared.yaml -# Generate a signing key -RUN generate_signing_key.py -o /conf/server.signing.key - WORKDIR /root # Copy the caddy config From ed25ea61686169a67f6724b2b58525b6a43b38a5 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Sat, 23 Jan 2021 02:18:39 +0000 Subject: [PATCH 6/6] Use all available worker types MatMaul and I were able to figure out why the media repository wasn't reachable. Turns out Complement was sending its traffic directly to the main process instead of nginx. The main process doesn't have the media resource available by default, so that tipped us off. Combined with the changes in https://github.com/matrix-org/synapse/pull/9162 outside requests to 8008 now actually get routed to workers if necessary. With this, the media tests now work - and we can switch on all available worker types! --- dockerfiles/SynapseWorkers.Dockerfile | 8 ++------ dockerfiles/synapse/caddy.complement.json | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/dockerfiles/SynapseWorkers.Dockerfile b/dockerfiles/SynapseWorkers.Dockerfile index bed8ceef..8214e27f 100644 --- a/dockerfiles/SynapseWorkers.Dockerfile +++ b/dockerfiles/SynapseWorkers.Dockerfile @@ -46,12 +46,8 @@ ENTRYPOINT \ SYNAPSE_REPORT_STATS=no \ # Set postgres authentication details which will be placed in the homeserver config file POSTGRES_PASSWORD=somesecret POSTGRES_USER=postgres POSTGRES_HOST=localhost \ - # Note: This list currently includes all worker types other than federation_sender, as - # Synapse fails to send federation transactions with it enabled. - # https://github.com/matrix-org/synapse/issues/9192 - SYNAPSE_WORKERS=pusher,user_dir,media_repository,appservice,synchrotron,federation_reader,federation_inbound \ - # To use all available workers: - #SYNAPSE_WORKERS=* \ + # Use all available worker types + SYNAPSE_WORKERS=* \ # Run the script that writes the necessary config files and starts supervisord, which in turn # starts everything else /configure_workers_and_start.py diff --git a/dockerfiles/synapse/caddy.complement.json b/dockerfiles/synapse/caddy.complement.json index 3373e809..db739c60 100644 --- a/dockerfiles/synapse/caddy.complement.json +++ b/dockerfiles/synapse/caddy.complement.json @@ -25,7 +25,7 @@ "handler": "reverse_proxy", "upstreams": [ { - "dial": "localhost:8080" + "dial": "localhost:8008" } ] }