Make running sytest docker work in worker mode

docker/Dockerfile

@@ -8,7 +8,7 @@ RUN apt-get -qq install -y \
     build-essential \
     perl \
     wget \
-    postgresql \
+    postgresql-9.6 \
     postgresql-client \
     libpq-dev \
     libssl-dev \

docker/Dockerfile-synapsepy2

@@ -1,14 +1,35 @@
 FROM matrixdotorg/sytest:latest
 
-RUN apt-get -qq install -y python python-dev python-virtualenv
+RUN apt-get -qq update
+RUN apt-get -qq install -y python python-dev python-virtualenv dos2unix eatmydata
 
 ENV PYTHON=python2
+ENV PGDATA=/var/lib/postgresql/data
+
+RUN su -c '/usr/lib/postgresql/9.6/bin/initdb -E "UTF-8" --lc-collate="en_US.UTF-8" --lc-ctype="en_US.UTF-8" --username=postgres' postgres
+
+# Turn off all the fsync stuff for postgres
+RUN mkdir -p /etc/postgresql/9.6/main/conf.d/
+RUN echo "fync=off" > /etc/postgresql/9.6/main/conf.d/fsync.conf
+RUN echo "full_page_writes=off" >> /etc/postgresql/9.6/main/conf.d/fsync.conf
 
 # /src is where we expect Synapse to be
 RUN mkdir /src
 
+# Create the virutal env upfront so we don't need to keep reinstall dependencies
+RUN $PYTHON -m virtualenv -p $PYTHON /venv/
+RUN /venv/bin/pip install -q --no-cache-dir matrix-synapse[all]
+RUN /venv/bin/pip install -q --no-cache-dir lxml psycopg2
+
+# Uninstall matrix-synapse package so it doesn't collide with the version we try
+# and test
+RUN /venv/bin/pip uninstall -q --no-cache-dir -y matrix-synapse
+
 # The dockerfile context, when ran by the buildscript, will actually be the
 # repo root, not the docker folder
 ADD docker/synapse_sytest.sh /synapse_sytest.sh
 RUN dos2unix /synapse_sytest.sh
+
+ADD docker/pydron.py /pydron.py
+
 ENTRYPOINT ["/synapse_sytest.sh"]

docker/Dockerfile-synapsepy3

@@ -1,14 +1,35 @@
 FROM matrixdotorg/sytest:latest
 
-RUN apt-get -qq install -y python3 python3-dev python3-virtualenv
+RUN apt-get -qq update
+RUN apt-get -qq install -y python3 python3-dev python3-virtualenv dos2unix eatmydata
 
 ENV PYTHON=python3
+ENV PGDATA=/var/lib/postgresql/data
+
+RUN su -c '/usr/lib/postgresql/9.6/bin/initdb -E "UTF-8" --lc-collate="en_US.UTF-8" --lc-ctype="en_US.UTF-8" --username=postgres' postgres
+
+# Turn off all the fsync stuff for postgres
+RUN mkdir -p /etc/postgresql/9.6/main/conf.d/
+RUN echo "fync=off" > /etc/postgresql/9.6/main/conf.d/fsync.conf
+RUN echo "full_page_writes=off" >> /etc/postgresql/9.6/main/conf.d/fsync.conf
 
 # /src is where we expect Synapse to be
 RUN mkdir /src
 
+# Create the virutal env upfront so we don't need to keep reinstall dependencies
+RUN $PYTHON -m virtualenv -p $PYTHON /venv/
+RUN /venv/bin/pip install -q --no-cache-dir matrix-synapse[all]
+RUN /venv/bin/pip install -q --no-cache-dir lxml psycopg2
+
+# Uninstall matrix-synapse package so it doesn't collide with the version we try
+# and test
+RUN /venv/bin/pip uninstall -q --no-cache-dir -y matrix-synapse
+
 # The dockerfile context, when ran by the buildscript, will actually be the
 # repo root, not the docker folder
 ADD docker/synapse_sytest.sh /synapse_sytest.sh
 RUN dos2unix /synapse_sytest.sh
-ENTRYPOINT ["/synapse_sytest.sh"]
+
+ADD docker/pydron.py /pydron.py
+
+ENTRYPOINT ["/synapse_sytest.sh"]

docker/README.md

@@ -1,6 +1,8 @@
 # SyTest Docker Images
 
-These Dockerfiles create containers for running SyTest in various configurations. SyTest is not included in these images, but its dependencies are.
+These Dockerfiles create containers for running SyTest in various
+configurations. SyTest is not included in these images, but its dependencies
+are.
 
 Included currently is:
 
@@ -16,13 +18,18 @@ Once pulled from Docker Hub, the container can be run on a Synapse checkout:
 $ docker run --rm -it -v /path/to/synapse\:/src -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapsepy2
 ```
 
-This will run on the same branch in SyTest as the Synapse checkout, if possible, but will fall back to using develop.
+This will run on the same branch in SyTest as the Synapse checkout, if possible,
+but will fall back to using develop.
 
-If you want to use an existing checkout of SyTest, mount it to `/test` inside the container by adding `-v /path/to/sytest\:/test` to the docker command.
+If you want to use an existing checkout of SyTest, mount it to `/test` inside
+the container by adding `-v /path/to/sytest\:/test` to the docker command (note
+that this will be written to).
 
-If you want to test against a PostgreSQL database, pass `-e POSTGRES=1` to the docker command.
+If you want to test against a PostgreSQL database, pass `-e POSTGRES=1` to the
+docker command.
 
-You can pass arguments to sytest by adding them at the end of the docker command.  For example, you can use
+You can pass arguments to sytest by adding them at the end of the docker
+command.  For example, you can use
 
 ```
 $ docker run --rm -it ... matrixdotorg/sytest-synapsepy2 tests/20profile-events.pl
@@ -32,7 +39,8 @@ to run only a single test.
 
 ## Building the containers
 
-The containers are built by executing `build.sh`. You will then have to push them up to Docker Hub:
+The containers are built by executing `build.sh`. You will then have to push
+them up to Docker Hub:
 
 ```
 $ docker push matrixdotorg/sytest

docker/pydron.py

@@ -17,28 +17,6 @@ def _print(out):
     sys.stderr.flush()
 
 
-def wait_for_start(process, url):
-    """
-    Wait for the start of Synapse by polling its HTTP endpoint.
-    """
-    tries = 100
-    while tries != 0:
-        tries -= 1
-
-        try:
-            r = requests.get("http://" + url.replace("http://", ""), timeout=5)
-            if r.status_code:
-                # We don't care what the status code is, just that it's
-                # responding.
-                return
-        except Exception as e:
-            time.sleep(0.5)
-            pass
-
-    process.kill()
-    raise ValueError("Never started, couldn't get %s!" % (url,))
-
-
 _print("Starting pydron...")
 
 # Sort the args into key:value pairs in a dict
@@ -65,72 +43,41 @@ def wait_for_start(process, url):
 # create a map for it.
 appname_map = {"user-directory": "user_dir"}
 
-# running is where we'll keep a map of app name to the Popen object (so we can
-# terminate it later).
-running = {}
 
 try:
+    _print("Starting main process")
+
     # First, try and start Synapse (which will set up the database).
     synapse = (
         args["--synapse-python"]
-        + " -m synapse.app.homeserver --config-path="
+        + " -m synapse.app.homeserver -D --config-path="
         + args["--synapse-config"]
     )
-    syn = subprocess.Popen(
-        shlex.split(synapse), stdout=subprocess.PIPE, stderr=subprocess.STDOUT
-    )
-    running["synapse"] = syn
-
-    # Wait for Synapse to start by polling its webserver until it responds.
-    wait_for_start(syn, urls["synapse"])
+    subprocess.run(shlex.split(synapse), check=True)
 
     # Then, start up all the workers.
     for i in configs.keys():
-
         # Get the synapse app name from the map, if needed. Otherwise, just
         # replace any dashes with underscores, so they match the Python module
         # name.
         appname = appname_map.get(i, i.replace("-", "_"))
 
+        _print("Starting %s" % (appname,))
+
         base = (
             args["--synapse-python"]
             + " -m synapse.app."
             + appname
+            + " -D --config-path="
+            + args["--synapse-config"]
             + " --config-path="
             + configs[i]
-            + " --config-path="
-            + args["--synapse-config"]
-        )
-        exc = subprocess.Popen(
-            shlex.split(base), stdout=subprocess.PIPE, stderr=subprocess.STDOUT
         )
-        running[i] = exc
-
-        # If we've been given a url for it, poll it until it's up
-        if i in urls:
-            wait_for_start(exc, urls[i])
-
-    # Check if any have outright failed to start up (syntax errors, etc). We do
-    # this even if we've waited for them to start, because not all workers have
-    # a URL.
-    failed_units = []
-
-    for x in running.keys():
-        if x in urls:
-            wait_for_start(running[x], urls[x])
-
-        code = running[x].poll()
-
-        if code:
-            _print("%s exited uncleanly with %s!" % (x, code))
-            failed_units.append(x)
-
-    if failed_units:
-        raise ValueError("failed units: %r" % (failed_units,))
+        subprocess.run(shlex.split(base), check=True)
 
     # Nothing failed, let's say we've started and then signal we're up by
     # serving the webserver.
-    print("Synapse started!")
+    _print("Synapse started!")
 
     class WebHandler(BaseHTTPRequestHandler):
         def do_GET(self):
@@ -143,6 +90,8 @@ def do_GET(self):
     server = HTTPServer(('', PORT_NUMBER), WebHandler)
     server.serve_forever()
 
+except KeyboardInterrupt:
+   pass
 except BaseException as e:
     # Log a traceback for debug
     import traceback
@@ -151,17 +100,7 @@ def do_GET(self):
     # If we get told to shut down, log it
     _print("Told to quit because %s" % (repr(e)))
 
-    # Terminate all the workers as cleanly as possible.
-    for x in running.keys():
-        if not running[x].returncode:
-            _print("Killing %s" % (x,))
-            running[x].kill()
-        else:
-            _print("%s was already dead" % (x,))
-
-    # If it's keyboard interrupt, exit cleanly -- sytest is finished.
-    if isinstance(e, KeyboardInterrupt):
-        sys.exit(0)
-
     # Otherwise, something bad has happened.
     sys.exit(1)
+finally:
+   subprocess.run(["pkill", "-9", "-f", "synapse.app"])

docker/synapse_sytest.sh

@@ -27,15 +27,12 @@ fi
 # PostgreSQL setup
 if [ -n "$POSTGRES" ]
 then
-
-    export PGDATA=/var/lib/postgresql/data
     export PGUSER=postgres
     export POSTGRES_DB_1=pg1
     export POSTGRES_DB_2=pg2
 
-    # Initialise the database files and start the database
-    su -c '/usr/lib/postgresql/9.6/bin/initdb -E "UTF-8" --lc-collate="en_US.UTF-8" --lc-ctype="en_US.UTF-8" --username=postgres' postgres
-    su -c '/usr/lib/postgresql/9.6/bin/pg_ctl -w -D /var/lib/postgresql/data start' postgres
+    # Start the database
+    su -c 'eatmydata /usr/lib/postgresql/9.6/bin/pg_ctl -w -D /var/lib/postgresql/data start' postgres
 
     # Use the Jenkins script to write out the configuration for a PostgreSQL using Synapse
     jenkins/prep_sytest_for_postgres.sh
@@ -46,25 +43,28 @@ then
 
 fi
 
-# Build the virtualenv, install extra deps that we will need for the tests
-$PYTHON -m virtualenv -p $PYTHON /venv/
+# We've already created the virtualenv, but lets double check we have all deps.
 /venv/bin/pip install -q --no-cache-dir -e /src/
-/venv/bin/pip install -q --no-cache-dir lxml psycopg2
 
 # Make sure all Perl deps are installed -- this is done in the docker build so will only install packages added since the last Docker build
 dos2unix ./install-deps.pl
 ./install-deps.pl
 
 # Run the tests
+>&2 echo "+++ Running tests"
+
 dos2unix ./run-tests.pl
 TEST_STATUS=0
+
 if [ -n "$WORKERS" ]
 then
-    ./run-tests.pl -I Synapse::ViaHaproxy --python=/venv/bin/python --dendron-binary=/test/docker/pydron.py -O tap --all "$@" > results.tap || TEST_STATUS=$?
+    ./run-tests.pl -I Synapse::ViaHaproxy --python=/venv/bin/python --dendron-binary=/pydron.py -O tap --all "$@" > results.tap || TEST_STATUS=$?
 else
     ./run-tests.pl -I Synapse --python=/venv/bin/python -O tap --all "$@" > results.tap || TEST_STATUS=$?
 fi
 
+>&2 echo "--- Copying assets"
+
 # Copy out the logs
 mkdir -p /logs
 cp results.tap /logs/results.tap