Faster smoketest. Fix Flaky works-offline gather (#506)

* faster smoketest using a custom config
* Network.enable must be called before offline is simulated.

Author: paulirish

lighthouse-cli/scripts/run-smoke-tests.sh

@@ -3,34 +3,36 @@
 cd lighthouse-cli/test/fixtures && python -m SimpleHTTPServer 9999 &
 
 NODE=$([ $(node -v | grep -E "v4") ] && echo "node --harmony" || echo "node")
-offline200result="URL responds with a 200 when offline"
+config="$PWD/lighthouse-cli/test/fixtures/smoketest-config.json"
+flags="--config-path=$config"
 
+offline200result="URL responds with a 200 when offline"
 
-$NODE lighthouse-cli http://localhost:9999/online-only.html > results
+$NODE lighthouse-cli $flags http://localhost:9999/online-only.html > results
 
 # test that we have results
 if ! grep -q "$offline200result" results; then
   echo "Fail! Lighthouse run didn't create a result file"
   exit 1
 fi
 
-# test that we have a meta viewport defined in the static page
+# test that we have a meta viewport defined on a static page
 if ! grep -q "HTML has a viewport <meta>: true" results; then
   echo "Fail! Meta viewort not detected in the page"
   exit 1
 fi
 
-# test static page which should fail the offline test
+# test a static page which should fail the offline test
 if ! grep -q "$offline200result: false" results; then
   echo "Fail! online only site worked while offline"
   cat results
   exit 1
 fi
 
-sleep 5s
+sleep 1s
 
 # test mojibrush which should pass the offline test
-$NODE  lighthouse-cli https://www.moji-brush.com > results
+$NODE lighthouse-cli $flags https://www.moji-brush.com > results
 
 if ! grep -q "$offline200result: true" results; then
   echo "Fail! offline ready site did not work while offline"

lighthouse-cli/test/fixtures/smoketest-config.json

@@ -0,0 +1,47 @@
+{
+  "passes": [{
+    "loadPage": true,
+    "gatherers": [
+      "service-worker",
+      "offline",
+      "viewport"
+    ]
+  }],
+
+  "audits": [
+    "service-worker",
+    "works-offline",
+    "viewport"
+  ],
+
+  "aggregations": [{
+    "name": "Progressive Web App",
+    "description": "These audits validate the aspects of a Progressive Web App.",
+    "scored": true,
+    "categorizable": true,
+    "items": [{
+      "name": "App can load on offline/flaky connections",
+      "description": "Ensuring your web app can respond when the network connection is unavailable or flaky is critical to providing your users a good experience. This is achieved through use of a <a href=\"https://developers.google.com/web/fundamentals/primers/service-worker/\">Service Worker</a>.",
+      "criteria": {
+        "service-worker": {
+          "rawValue": true,
+          "weight": 1
+        },
+        "works-offline": {
+          "rawValue": true,
+          "weight": 1
+        }
+      }
+    }, {
+      "name": "Design is mobile-friendly",
+      "description": "Users increasingly experience your app on mobile devices, so it's important to ensure that the experience can adapt to smaller screens.",
+      "criteria": {
+        "viewport": {
+          "rawValue": true,
+          "weight": 1
+        }
+      }
+    }
+    ]
+  }]
+}

lighthouse-core/driver/gatherers/offline.js

@@ -38,23 +38,25 @@ const requestPage = function() {
 
 class Offline extends Gather {
 
-  static goOffline(driver) {
-    return driver.sendCommand('Network.emulateNetworkConditions', {
-      offline: true,
+  static config(opts) {
+    return {
+      offline: opts.offline,
       // values of 0 remove any active throttling. crbug.com/456324#c9
       latency: 0,
       downloadThroughput: 0,
       uploadThroughput: 0
+    };
+  }
+
+  static goOffline(driver) {
+    // Network.enable must be called for Network.emulateNetworkConditions to work
+    return driver.sendCommand('Network.enable').then(_ => {
+      driver.sendCommand('Network.emulateNetworkConditions', Offline.config({offline: true}));
     });
   }
 
   static goOnline(driver) {
-    return driver.sendCommand('Network.emulateNetworkConditions', {
-      offline: false,
-      latency: 0,
-      downloadThroughput: 0,
-      uploadThroughput: 0
-    });
+    return driver.sendCommand('Network.emulateNetworkConditions', Offline.config({offline: false}));
   }
 
   afterPass(options) {

lighthouse-core/lib/emulation.js

@@ -73,6 +73,8 @@ function enableNexus5X(driver) {
 
   return Promise.all([
     driver.sendCommand('Emulation.setDeviceMetricsOverride', NEXUS5X_EMULATION_METRICS),
+    // Network.enable must be called for UA overriding to work
+    driver.sendCommand('Network.enable'),
     driver.sendCommand('Network.setUserAgentOverride', NEXUS5X_USERAGENT),
     driver.sendCommand('Emulation.setTouchEmulationEnabled', {
       enabled: true,