From 07f29647b7f5ab66ed865e846c15fa0b98361688 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 13 Mar 2019 17:26:29 +0000 Subject: [PATCH 1/2] Make running sytest docker work in worker mode This also includes moving a bunch of setup into the docker image so that the containers start up a lot faster. --- docker/Dockerfile | 2 +- docker/Dockerfile-synapsepy2 | 23 ++++++++- docker/Dockerfile-synapsepy3 | 25 ++++++++- docker/README.md | 2 +- docker/pydron.py | 89 +++++--------------------------- docker/synapse_sytest.sh | 18 +++---- lib/SyTest/Homeserver/Synapse.pm | 27 ++++++++-- 7 files changed, 92 insertions(+), 94 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0e5e3c727..b8c868152 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get -qq install -y \ build-essential \ perl \ wget \ - postgresql \ + postgresql-9.6 \ postgresql-client \ libpq-dev \ libssl-dev \ diff --git a/docker/Dockerfile-synapsepy2 b/docker/Dockerfile-synapsepy2 index 79ace51e8..fd8381c76 100644 --- a/docker/Dockerfile-synapsepy2 +++ b/docker/Dockerfile-synapsepy2 @@ -1,14 +1,35 @@ FROM matrixdotorg/sytest:latest -RUN apt-get -qq install -y python python-dev python-virtualenv +RUN apt-get -qq update +RUN apt-get -qq install -y python python-dev python-virtualenv dos2unix eatmydata ENV PYTHON=python2 +ENV PGDATA=/var/lib/postgresql/data + +RUN su -c '/usr/lib/postgresql/9.6/bin/initdb -E "UTF-8" --lc-collate="en_US.UTF-8" --lc-ctype="en_US.UTF-8" --username=postgres' postgres + +# Turn of all the fsync stuff for postgres +RUN mkdir -p /etc/postgresql/9.6/main/conf.d/ +RUN echo "fync=off" > /etc/postgresql/9.6/main/conf.d/fsync.conf +RUN echo "full_page_writes=off" >> /etc/postgresql/9.6/main/conf.d/fsync.conf # /src is where we expect Synapse to be RUN mkdir /src +# Create the virutal env upfront so we don't need to keep reinstall dependencies +RUN $PYTHON -m virtualenv -p $PYTHON /venv/ +RUN /venv/bin/pip install -q --no-cache-dir matrix-synapse[all] +RUN /venv/bin/pip install -q --no-cache-dir lxml psycopg2 + +# Uninstall matrix-synapse package so it doesn't collide with the version we try +# Zand test +RUN /venv/bin/pip uninstall -q --no-cache-dir -y matrix-synapse + # The dockerfile context, when ran by the buildscript, will actually be the # repo root, not the docker folder ADD docker/synapse_sytest.sh /synapse_sytest.sh RUN dos2unix /synapse_sytest.sh + +ADD docker/pydron.py /pydron.py + ENTRYPOINT ["/synapse_sytest.sh"] diff --git a/docker/Dockerfile-synapsepy3 b/docker/Dockerfile-synapsepy3 index a20f70178..15621fc11 100644 --- a/docker/Dockerfile-synapsepy3 +++ b/docker/Dockerfile-synapsepy3 @@ -1,14 +1,35 @@ FROM matrixdotorg/sytest:latest -RUN apt-get -qq install -y python3 python3-dev python3-virtualenv +RUN apt-get -qq update +RUN apt-get -qq install -y python3 python3-dev python3-virtualenv dos2unix eatmydata ENV PYTHON=python3 +ENV PGDATA=/var/lib/postgresql/data + +RUN su -c '/usr/lib/postgresql/9.6/bin/initdb -E "UTF-8" --lc-collate="en_US.UTF-8" --lc-ctype="en_US.UTF-8" --username=postgres' postgres + +# Turn of all the fsync stuff for postgres +RUN mkdir -p /etc/postgresql/9.6/main/conf.d/ +RUN echo "fync=off" > /etc/postgresql/9.6/main/conf.d/fsync.conf +RUN echo "full_page_writes=off" >> /etc/postgresql/9.6/main/conf.d/fsync.conf # /src is where we expect Synapse to be RUN mkdir /src +# Create the virutal env upfront so we don't need to keep reinstall dependencies +RUN $PYTHON -m virtualenv -p $PYTHON /venv/ +RUN /venv/bin/pip install -q --no-cache-dir matrix-synapse[all] +RUN /venv/bin/pip install -q --no-cache-dir lxml psycopg2 + +# Uninstall matrix-synapse package so it doesn't collide with the version we try +# Zand test +RUN /venv/bin/pip uninstall -q --no-cache-dir -y matrix-synapse + # The dockerfile context, when ran by the buildscript, will actually be the # repo root, not the docker folder ADD docker/synapse_sytest.sh /synapse_sytest.sh RUN dos2unix /synapse_sytest.sh -ENTRYPOINT ["/synapse_sytest.sh"] \ No newline at end of file + +ADD docker/pydron.py /pydron.py + +ENTRYPOINT ["/synapse_sytest.sh"] diff --git a/docker/README.md b/docker/README.md index f3a3d3800..cd6af1bc6 100644 --- a/docker/README.md +++ b/docker/README.md @@ -18,7 +18,7 @@ $ docker run --rm -it -v /path/to/synapse\:/src -v /path/to/where/you/want/logs\ This will run on the same branch in SyTest as the Synapse checkout, if possible, but will fall back to using develop. -If you want to use an existing checkout of SyTest, mount it to `/test` inside the container by adding `-v /path/to/sytest\:/test` to the docker command. +If you want to use an existing checkout of SyTest, mount it to `/test` inside the container by adding `-v /path/to/sytest\:/test` to the docker command (note that this will be written to). If you want to test against a PostgreSQL database, pass `-e POSTGRES=1` to the docker command. diff --git a/docker/pydron.py b/docker/pydron.py index 3a2575ec0..bee9f4991 100755 --- a/docker/pydron.py +++ b/docker/pydron.py @@ -17,28 +17,6 @@ def _print(out): sys.stderr.flush() -def wait_for_start(process, url): - """ - Wait for the start of Synapse by polling its HTTP endpoint. - """ - tries = 100 - while tries != 0: - tries -= 1 - - try: - r = requests.get("http://" + url.replace("http://", ""), timeout=5) - if r.status_code: - # We don't care what the status code is, just that it's - # responding. - return - except Exception as e: - time.sleep(0.5) - pass - - process.kill() - raise ValueError("Never started, couldn't get %s!" % (url,)) - - _print("Starting pydron...") # Sort the args into key:value pairs in a dict @@ -65,72 +43,41 @@ def wait_for_start(process, url): # create a map for it. appname_map = {"user-directory": "user_dir"} -# running is where we'll keep a map of app name to the Popen object (so we can -# terminate it later). -running = {} try: + _print("Starting main process") + # First, try and start Synapse (which will set up the database). synapse = ( args["--synapse-python"] - + " -m synapse.app.homeserver --config-path=" + + " -m synapse.app.homeserver -D --config-path=" + args["--synapse-config"] ) - syn = subprocess.Popen( - shlex.split(synapse), stdout=subprocess.PIPE, stderr=subprocess.STDOUT - ) - running["synapse"] = syn - - # Wait for Synapse to start by polling its webserver until it responds. - wait_for_start(syn, urls["synapse"]) + subprocess.run(shlex.split(synapse), check=True) # Then, start up all the workers. for i in configs.keys(): - # Get the synapse app name from the map, if needed. Otherwise, just # replace any dashes with underscores, so they match the Python module # name. appname = appname_map.get(i, i.replace("-", "_")) + _print("Starting %s" % (appname,)) + base = ( args["--synapse-python"] + " -m synapse.app." + appname + + " -D --config-path=" + + args["--synapse-config"] + " --config-path=" + configs[i] - + " --config-path=" - + args["--synapse-config"] - ) - exc = subprocess.Popen( - shlex.split(base), stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) - running[i] = exc - - # If we've been given a url for it, poll it until it's up - if i in urls: - wait_for_start(exc, urls[i]) - - # Check if any have outright failed to start up (syntax errors, etc). We do - # this even if we've waited for them to start, because not all workers have - # a URL. - failed_units = [] - - for x in running.keys(): - if x in urls: - wait_for_start(running[x], urls[x]) - - code = running[x].poll() - - if code: - _print("%s exited uncleanly with %s!" % (x, code)) - failed_units.append(x) - - if failed_units: - raise ValueError("failed units: %r" % (failed_units,)) + subprocess.run(shlex.split(base), check=True) # Nothing failed, let's say we've started and then signal we're up by # serving the webserver. - print("Synapse started!") + _print("Synapse started!") class WebHandler(BaseHTTPRequestHandler): def do_GET(self): @@ -143,6 +90,8 @@ def do_GET(self): server = HTTPServer(('', PORT_NUMBER), WebHandler) server.serve_forever() +except KeyboardInterrupt: + pass except BaseException as e: # Log a traceback for debug import traceback @@ -151,17 +100,7 @@ def do_GET(self): # If we get told to shut down, log it _print("Told to quit because %s" % (repr(e))) - # Terminate all the workers as cleanly as possible. - for x in running.keys(): - if not running[x].returncode: - _print("Killing %s" % (x,)) - running[x].kill() - else: - _print("%s was already dead" % (x,)) - - # If it's keyboard interrupt, exit cleanly -- sytest is finished. - if isinstance(e, KeyboardInterrupt): - sys.exit(0) - # Otherwise, something bad has happened. sys.exit(1) +finally: + subprocess.run(["pkill", "-9", "-f", "synapse.app"]) diff --git a/docker/synapse_sytest.sh b/docker/synapse_sytest.sh index aa0f0b37c..8697f0f90 100755 --- a/docker/synapse_sytest.sh +++ b/docker/synapse_sytest.sh @@ -27,15 +27,12 @@ fi # PostgreSQL setup if [ -n "$POSTGRES" ] then - - export PGDATA=/var/lib/postgresql/data export PGUSER=postgres export POSTGRES_DB_1=pg1 export POSTGRES_DB_2=pg2 - # Initialise the database files and start the database - su -c '/usr/lib/postgresql/9.6/bin/initdb -E "UTF-8" --lc-collate="en_US.UTF-8" --lc-ctype="en_US.UTF-8" --username=postgres' postgres - su -c '/usr/lib/postgresql/9.6/bin/pg_ctl -w -D /var/lib/postgresql/data start' postgres + # Start the database + su -c 'eatmydata /usr/lib/postgresql/9.6/bin/pg_ctl -w -D /var/lib/postgresql/data start' postgres # Use the Jenkins script to write out the configuration for a PostgreSQL using Synapse jenkins/prep_sytest_for_postgres.sh @@ -46,25 +43,28 @@ then fi -# Build the virtualenv, install extra deps that we will need for the tests -$PYTHON -m virtualenv -p $PYTHON /venv/ +# We've already created the virtualenv, but lets double check we have all deps. /venv/bin/pip install -q --no-cache-dir -e /src/ -/venv/bin/pip install -q --no-cache-dir lxml psycopg2 # Make sure all Perl deps are installed -- this is done in the docker build so will only install packages added since the last Docker build dos2unix ./install-deps.pl ./install-deps.pl # Run the tests +>&2 echo "+++ Running tests" + dos2unix ./run-tests.pl TEST_STATUS=0 + if [ -n "$WORKERS" ] then - ./run-tests.pl -I Synapse::ViaHaproxy --python=/venv/bin/python --dendron-binary=/test/docker/pydron.py -O tap --all "$@" > results.tap || TEST_STATUS=$? + ./run-tests.pl -I Synapse::ViaHaproxy --python=/venv/bin/python --dendron-binary=/pydron.py -O tap --all "$@" > results.tap || TEST_STATUS=$? else ./run-tests.pl -I Synapse --python=/venv/bin/python -O tap --all "$@" > results.tap || TEST_STATUS=$? fi +>&2 echo "--- Copying assets" + # Copy out the logs mkdir -p /logs cp results.tap /logs/results.tap diff --git a/lib/SyTest/Homeserver/Synapse.pm b/lib/SyTest/Homeserver/Synapse.pm index 7a6fab313..37361a394 100644 --- a/lib/SyTest/Homeserver/Synapse.pm +++ b/lib/SyTest/Homeserver/Synapse.pm @@ -145,6 +145,9 @@ sub start my $macaroon_secret_key = "secret_$port"; my $registration_shared_secret = "reg_secret"; + my $cert = $self->{paths}{cert_file} = "$cwd/keys/tls-selfsigned.crt"; + my $key = $self->{paths}{key_file} = "$cwd/keys/tls-selfsigned.key"; + my $config_path = $self->{paths}{config} = $self->write_yaml_file( "config.yaml" => { server_name => $self->server_name, log_file => "$log", @@ -162,6 +165,8 @@ sub start macaroon_secret_key => $macaroon_secret_key, registration_shared_secret => $registration_shared_secret, + pid_file => "$hs_dir/homeserver.pid", + use_frozen_events => "true", allow_guest_access => "True", @@ -595,6 +600,7 @@ sub wrap_synapse_command my $bind_host = $self->{bind_host}; my $log = $self->{paths}{log}; + my $hsdir = $self->{hs_dir}; -x $self->{dendron} or die "Cannot exec($self->{dendron}) - $!"; @@ -612,6 +618,7 @@ sub wrap_synapse_command { my $pusher_config_path = $self->write_yaml_file( "pusher.yaml" => { "worker_app" => "synapse.app.pusher", + "worker_pid_file" => "$hsdir/pusher.pid", "worker_log_file" => "$log.pusher", "worker_replication_host" => "$bind_host", "worker_replication_port" => $self->{ports}{synapse_replication_tcp}, @@ -636,6 +643,7 @@ sub wrap_synapse_command { my $appservice_config_path = $self->write_yaml_file( "appservice.yaml" => { "worker_app" => "synapse.app.appservice", + "worker_pid_file" => "$hsdir/appservice.pid", "worker_log_file" => "$log.appservice", "worker_replication_host" => "$bind_host", "worker_replication_port" => $self->{ports}{synapse_replication_tcp}, @@ -660,6 +668,7 @@ sub wrap_synapse_command { my $federation_sender_config_path = $self->write_yaml_file( "federation_sender.yaml" => { "worker_app" => "synapse.app.federation_sender", + "worker_pid_file" => "$hsdir/federation_sender.pid", "worker_log_file" => "$log.federation_sender", "worker_replication_host" => "$bind_host", "worker_replication_port" => $self->{ports}{synapse_replication_tcp}, @@ -684,6 +693,7 @@ sub wrap_synapse_command { my $synchrotron_config_path = $self->write_yaml_file( "synchrotron.yaml" => { "worker_app" => "synapse.app.synchrotron", + "worker_pid_file" => "$hsdir/synchrotron.pid", "worker_log_file" => "$log.synchrotron", "worker_replication_host" => "$bind_host", "worker_replication_port" => $self->{ports}{synapse_replication_tcp}, @@ -716,8 +726,10 @@ sub wrap_synapse_command { my $federation_reader_config_path = $self->write_yaml_file( "federation_reader.yaml" => { "worker_app" => "synapse.app.federation_reader", + "worker_pid_file" => "$hsdir/federation_reader.pid", "worker_log_file" => "$log.federation_reader", "worker_replication_host" => "$bind_host", + "worker_replication_http_port" => $self->{ports}{synapse_unsecure}, "worker_replication_port" => $self->{ports}{synapse_replication_tcp}, "worker_listeners" => [ { @@ -748,6 +760,7 @@ sub wrap_synapse_command { my $media_repository_config_path = $self->write_yaml_file( "media_repository.yaml" => { "worker_app" => "synapse.app.media_repository", + "worker_pid_file" => "$hsdir/media_repository.pid", "worker_log_file" => "$log.media_repository", "worker_replication_host" => "$bind_host", "worker_replication_port" => $self->{ports}{synapse_replication_tcp}, @@ -779,11 +792,13 @@ sub wrap_synapse_command { my $client_reader_config_path = $self->write_yaml_file( "client_reader.yaml" => { - "worker_app" => "synapse.app.client_reader", - "worker_log_file" => "$log.client_reader", - "worker_replication_host" => "$bind_host", - "worker_replication_port" => $self->{ports}{synapse_replication_tcp}, - "worker_listeners" => [ + "worker_app" => "synapse.app.client_reader", + "worker_pid_file" => "$hsdir/client_reader.pid", + "worker_log_file" => "$log.client_reader", + "worker_replication_host" => "$bind_host", + "worker_replication_http_port" => $self->{ports}{synapse_unsecure}, + "worker_replication_port" => $self->{ports}{synapse_replication_tcp}, + "worker_listeners" => [ { type => "http", resources => [{ names => ["client"] }], @@ -812,6 +827,7 @@ sub wrap_synapse_command { my $user_dir_config_path = $self->write_yaml_file( "user_dir.yaml" => { "worker_app" => "synapse.app.user_dir", + "worker_pid_file" => "$hsdir/user_dir.pid", "worker_log_file" => "$log.user_dir", "worker_replication_host" => "$bind_host", "worker_replication_port" => $self->{ports}{synapse_replication_tcp}, @@ -844,6 +860,7 @@ sub wrap_synapse_command { my $event_creator_config_path = $self->write_yaml_file( "event_creator.yaml" => { "worker_app" => "synapse.app.event_creator", + "worker_pid_file" => "$hsdir/event_creator.pid", "worker_log_file" => "$log.event_creator", "worker_replication_host" => "$bind_host", "worker_replication_port" => $self->{ports}{synapse_replication_tcp}, From 876bbd585ed1da5ac62a9586e43b8d63fa5ca0e3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 15 Mar 2019 13:22:13 +0000 Subject: [PATCH 2/2] Fixup comments and docs --- docker/Dockerfile-synapsepy2 | 4 ++-- docker/Dockerfile-synapsepy3 | 4 ++-- docker/README.md | 20 ++++++++++++++------ 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/docker/Dockerfile-synapsepy2 b/docker/Dockerfile-synapsepy2 index fd8381c76..e2218fb85 100644 --- a/docker/Dockerfile-synapsepy2 +++ b/docker/Dockerfile-synapsepy2 @@ -8,7 +8,7 @@ ENV PGDATA=/var/lib/postgresql/data RUN su -c '/usr/lib/postgresql/9.6/bin/initdb -E "UTF-8" --lc-collate="en_US.UTF-8" --lc-ctype="en_US.UTF-8" --username=postgres' postgres -# Turn of all the fsync stuff for postgres +# Turn off all the fsync stuff for postgres RUN mkdir -p /etc/postgresql/9.6/main/conf.d/ RUN echo "fync=off" > /etc/postgresql/9.6/main/conf.d/fsync.conf RUN echo "full_page_writes=off" >> /etc/postgresql/9.6/main/conf.d/fsync.conf @@ -22,7 +22,7 @@ RUN /venv/bin/pip install -q --no-cache-dir matrix-synapse[all] RUN /venv/bin/pip install -q --no-cache-dir lxml psycopg2 # Uninstall matrix-synapse package so it doesn't collide with the version we try -# Zand test +# and test RUN /venv/bin/pip uninstall -q --no-cache-dir -y matrix-synapse # The dockerfile context, when ran by the buildscript, will actually be the diff --git a/docker/Dockerfile-synapsepy3 b/docker/Dockerfile-synapsepy3 index 15621fc11..3e6bf0705 100644 --- a/docker/Dockerfile-synapsepy3 +++ b/docker/Dockerfile-synapsepy3 @@ -8,7 +8,7 @@ ENV PGDATA=/var/lib/postgresql/data RUN su -c '/usr/lib/postgresql/9.6/bin/initdb -E "UTF-8" --lc-collate="en_US.UTF-8" --lc-ctype="en_US.UTF-8" --username=postgres' postgres -# Turn of all the fsync stuff for postgres +# Turn off all the fsync stuff for postgres RUN mkdir -p /etc/postgresql/9.6/main/conf.d/ RUN echo "fync=off" > /etc/postgresql/9.6/main/conf.d/fsync.conf RUN echo "full_page_writes=off" >> /etc/postgresql/9.6/main/conf.d/fsync.conf @@ -22,7 +22,7 @@ RUN /venv/bin/pip install -q --no-cache-dir matrix-synapse[all] RUN /venv/bin/pip install -q --no-cache-dir lxml psycopg2 # Uninstall matrix-synapse package so it doesn't collide with the version we try -# Zand test +# and test RUN /venv/bin/pip uninstall -q --no-cache-dir -y matrix-synapse # The dockerfile context, when ran by the buildscript, will actually be the diff --git a/docker/README.md b/docker/README.md index cd6af1bc6..54ea64f35 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,6 +1,8 @@ # SyTest Docker Images -These Dockerfiles create containers for running SyTest in various configurations. SyTest is not included in these images, but its dependencies are. +These Dockerfiles create containers for running SyTest in various +configurations. SyTest is not included in these images, but its dependencies +are. Included currently is: @@ -16,13 +18,18 @@ Once pulled from Docker Hub, the container can be run on a Synapse checkout: $ docker run --rm -it -v /path/to/synapse\:/src -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapsepy2 ``` -This will run on the same branch in SyTest as the Synapse checkout, if possible, but will fall back to using develop. +This will run on the same branch in SyTest as the Synapse checkout, if possible, +but will fall back to using develop. -If you want to use an existing checkout of SyTest, mount it to `/test` inside the container by adding `-v /path/to/sytest\:/test` to the docker command (note that this will be written to). +If you want to use an existing checkout of SyTest, mount it to `/test` inside +the container by adding `-v /path/to/sytest\:/test` to the docker command (note +that this will be written to). -If you want to test against a PostgreSQL database, pass `-e POSTGRES=1` to the docker command. +If you want to test against a PostgreSQL database, pass `-e POSTGRES=1` to the +docker command. -You can pass arguments to sytest by adding them at the end of the docker command. For example, you can use +You can pass arguments to sytest by adding them at the end of the docker +command. For example, you can use ``` $ docker run --rm -it ... matrixdotorg/sytest-synapsepy2 tests/20profile-events.pl @@ -32,7 +39,8 @@ to run only a single test. ## Building the containers -The containers are built by executing `build.sh`. You will then have to push them up to Docker Hub: +The containers are built by executing `build.sh`. You will then have to push +them up to Docker Hub: ``` $ docker push matrixdotorg/sytest