diff --git a/synapse/app/_base.py b/synapse/app/_base.py
index 62c633146fe6..79da72db60ad 100644
--- a/synapse/app/_base.py
+++ b/synapse/app/_base.py
@@ -213,12 +213,17 @@ def refresh_certificate(hs):
     Refresh the TLS certificates that Synapse is using by re-reading them from
     disk and updating the TLS context factories to use them.
     """
-    logging.info("Loading certificate from disk...")
-    hs.config.read_certificate_from_disk()
-    hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config)
     hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory(
         hs.config
     )
+
+    if hs.config.no_tls:
+        logging.info("Serving TLS is disabled, not loading certificates.")
+        return
+
+    logging.info("Loading certificate from disk...")
+    hs.config.read_certificate_from_disk()
+    hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config)
     logging.info("Certificate loaded.")
 
     if hs._listening_services:
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index b4476bf16e36..a67937d09c11 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -91,9 +91,11 @@ def _listener_http(self, config, listener_config):
         site_tag = listener_config.get("tag", port)
 
         if tls and config.no_tls:
-            raise ConfigError(
-                "Listener on port %i has TLS enabled, but no_tls is set" % (port,),
+            logger.info(
+                "Listener on port %i has TLS enabled, but no_tls is set, skipping",
+                port,
             )
+            return
 
         resources = {}
         for res in listener_config["resources"]:
diff --git a/synapse/config/tls.py b/synapse/config/tls.py
index 9fcc79816dc0..89ecc8a56436 100644
--- a/synapse/config/tls.py
+++ b/synapse/config/tls.py
@@ -122,10 +122,7 @@ def read_certificate_from_disk(self):
                 )
             )
 
-        if not self.no_tls:
-            self.tls_private_key = self.read_tls_private_key(self.tls_private_key_file)
-
-        self.tls_fingerprints = list(self._original_tls_fingerprints)
+        self.tls_private_key = self.read_tls_private_key(self.tls_private_key_file)
 
         # Check that our own certificate is included in the list of fingerprints
         # and include it if it is not.