Adds support for custom audits and gatherers

Author: paullewis

custom/audit.js

@@ -0,0 +1,19 @@
+/**
+ *
+ * Copyright 2016 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Pass-through file for developer convenience.
+module.exports = require('../lighthouse-core/audits/audit');

custom/gatherer.js

@@ -0,0 +1,19 @@
+/**
+ *
+ * Copyright 2016 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Pass-through file for developer convenience.
+module.exports = require('../lighthouse-core/gather/gatherers/gatherer');

lighthouse-core/audits/audit.js

@@ -33,6 +33,13 @@ class Audit {
     throw new Error('Audit meta information must be overridden.');
   }
 
+  /**
+   * @throws
+   */
+  static audit() {
+    throw new Error('Audit audit() function must be overridden.');
+  }
+
   /**
    * @param {!AuditResultInput} result
    * @return {!AuditResult}

lighthouse-core/config/index.js

@@ -23,6 +23,7 @@ const SpeedlineGatherer = require('../gather/gatherers/speedline');
 
 const GatherRunner = require('../gather/gather-runner');
 const log = require('../lib/log');
+const path = require('path');
 
 // cleanTrace is run to remove duplicate TracingStartedInPage events,
 // and to change TracingStartedInBrowser events into TracingStartedInPage.
@@ -107,7 +108,7 @@ function cleanTrace(trace) {
   return trace;
 }
 
-function filterPasses(passes, audits) {
+function filterPasses(passes, audits, paths) {
   const requiredGatherers = getGatherersNeededByAudits(audits);
 
   // Make sure we only have the gatherers that are needed by the audits
@@ -117,7 +118,7 @@ function filterPasses(passes, audits) {
 
     freshPass.gatherers = freshPass.gatherers.filter(gatherer => {
       try {
-        const GathererClass = GatherRunner.getGathererClass(gatherer);
+        const GathererClass = GatherRunner.getGathererClass(gatherer, paths);
         return requiredGatherers.has(GathererClass.name);
       } catch (requireError) {
         throw new Error(`Unable to locate gatherer: ${gatherer}`);
@@ -171,16 +172,79 @@ function filterAudits(audits, auditWhitelist) {
   return filteredAudits;
 }
 
-function expandAudits(audits) {
+function expandAudits(audits, paths) {
+  const rootPath = path.join(__dirname, '../../');
+
   return audits.map(audit => {
-    try {
-      return require(`../audits/${audit}`);
-    } catch (requireError) {
+    // Check each path to see if the audit can be located. First match wins.
+    const AuditClass = paths.reduce((definition, auditPath) => {
+      // If the definition has already been found, just propagate it. Otherwise try a search
+      // on the path in this iteration of the loop.
+      if (definition !== null) {
+        return definition;
+      }
+
+      const requirePath = auditPath.startsWith('/') ? auditPath : path.join(rootPath, auditPath);
+      try {
+        return require(`${requirePath}/${audit}`);
+      } catch (requireError) {
+        return null;
+      }
+    }, null);
+
+    if (!AuditClass) {
       throw new Error(`Unable to locate audit: ${audit}`);
     }
+
+    // Confirm that the audit appears valid.
+    const auditValidation = validateAudit(AuditClass);
+    if (!auditValidation.valid) {
+      const errors = Object.keys(auditValidation)
+          .reduce((errorList, item) => {
+            // Ignore the valid property as it's generated from the other items in the object.
+            if (item === 'valid') {
+              return errorList;
+            }
+
+            return errorList + (auditValidation[item] ? '' : `\n - ${item} is missing`);
+          }, '');
+
+      throw new Error(`Invalid audit class: ${errors}`);
+    }
+
+    return AuditClass;
   });
 }
 
+function validateAudit(auditDefinition) {
+  const hasAuditMethod = typeof auditDefinition.audit === 'function';
+  const hasMeta = typeof auditDefinition.meta === 'object';
+  const hasMetaName = hasMeta && typeof auditDefinition.meta.name !== 'undefined';
+  const hasMetaCategory = hasMeta && typeof auditDefinition.meta.category !== 'undefined';
+  const hasMetaDescription = hasMeta && typeof auditDefinition.meta.description !== 'undefined';
+  const hasMetaRequiredArtifacts = hasMeta && Array.isArray(auditDefinition.meta.requiredArtifacts);
+  const hasGenerateAuditResult = typeof auditDefinition.generateAuditResult === 'function';
+
+  return {
+    'valid': (
+      hasAuditMethod &&
+      hasMeta &&
+      hasMetaName &&
+      hasMetaCategory &&
+      hasMetaDescription &&
+      hasMetaRequiredArtifacts &&
+      hasGenerateAuditResult
+    ),
+    'audit()': hasAuditMethod,
+    'meta property': hasMeta,
+    'meta.name property': hasMetaName,
+    'meta.category property': hasMetaCategory,
+    'meta.description property': hasMetaDescription,
+    'meta.requiredArtifacts array': hasMetaRequiredArtifacts,
+    'generateAuditResult()': hasGenerateAuditResult
+  };
+}
+
 function expandArtifacts(artifacts, includeSpeedline) {
   const expandedArtifacts = Object.assign({}, artifacts);
 
@@ -241,19 +305,53 @@ class Config {
       configJSON = defaultConfig;
     }
 
-    this._audits = configJSON.audits ? expandAudits(
-        filterAudits(configJSON.audits, auditWhitelist)
+    this._configJSON = this._initRequirePaths(configJSON);
+
+    this._audits = this.json.audits ? expandAudits(
+        filterAudits(this.json.audits, auditWhitelist), this.json.paths.audits
         ) : null;
     // filterPasses expects audits to have been expanded
-    this._passes = configJSON.passes ? filterPasses(configJSON.passes, this._audits) : null;
-    this._auditResults = configJSON.auditResults ? Array.from(configJSON.auditResults) : null;
+    this._passes = this.json.passes ?
+        filterPasses(this.json.passes, this._audits, this.json.paths.gatherers) :
+        null;
+    this._auditResults = this.json.auditResults ? Array.from(this.json.auditResults) : null;
     this._artifacts = null;
-    if (configJSON.artifacts) {
-      this._artifacts = expandArtifacts(configJSON.artifacts,
+    if (this.json.artifacts) {
+      this._artifacts = expandArtifacts(this.json.artifacts,
           // If time-to-interactive is present, add the speedline artifact
-          configJSON.audits && configJSON.audits.find(a => a === 'time-to-interactive'));
+          this.json.audits && this.json.audits.find(a => a === 'time-to-interactive'));
+    }
+    this._aggregations = this.json.aggregations ? Array.from(this.json.aggregations) : null;
+  }
+
+  _initRequirePaths(configJSON) {
+    if (typeof configJSON.paths !== 'object') {
+      configJSON.paths = {};
+    }
+
+    if (!Array.isArray(configJSON.paths.audits)) {
+      configJSON.paths.audits = [];
+    }
+
+    if (!Array.isArray(configJSON.paths.gatherers)) {
+      configJSON.paths.gatherers = [];
     }
-    this._aggregations = configJSON.aggregations ? Array.from(configJSON.aggregations) : null;
+
+    // Make sure the default paths are prepended to the list
+    if (configJSON.paths.audits.indexOf('lighthouse-core/audits') === -1) {
+      configJSON.paths.audits.unshift('lighthouse-core/audits');
+    }
+
+    if (configJSON.paths.gatherers.indexOf('lighthouse-core/gather/gatherers') === -1) {
+      configJSON.paths.gatherers.unshift('lighthouse-core/gather/gatherers');
+    }
+
+    return configJSON;
+  }
+
+  /** @type {!Object} */
+  get json() {
+    return this._configJSON;
   }
 
   /** @type {Array<!Pass>} */

lighthouse-core/gather/gather-runner.js

@@ -18,6 +18,7 @@
 
 const log = require('../lib/log.js');
 const Audit = require('../audits/audit');
+const path = require('path');
 
 /**
  * Class that drives browser to load the page and runs gatherer lifecycle hooks.
@@ -189,7 +190,31 @@ class GatherRunner {
     const driver = options.driver;
     const tracingData = {traces: {}};
 
-    passes = GatherRunner.instantiateGatherers(passes);
+    if (typeof options.url !== 'string' || options.url.length === 0) {
+      return Promise.reject(new Error('You must provide a url to the driver'));
+    }
+
+    if (typeof options.flags === 'undefined') {
+      options.flags = {};
+    }
+
+    if (typeof options.config === 'undefined') {
+      return Promise.reject(new Error('You must provide a config'));
+    }
+
+    const configJSON = options.config.json;
+
+    // Default mobile emulation and page loading to true.
+    // The extension will switch these off initially.
+    if (typeof options.flags.mobile === 'undefined') {
+      options.flags.mobile = true;
+    }
+
+    if (typeof options.flags.loadPage === 'undefined') {
+      options.flags.loadPage = true;
+    }
+
+    passes = this.instantiateGatherers(passes, configJSON.paths.gatherers);
 
     return driver.connect()
       .then(_ => GatherRunner.setupDriver(driver, options))
@@ -222,26 +247,55 @@ class GatherRunner {
         const artifacts = Object.assign({}, tracingData);
         passes.forEach(pass => {
           pass.gatherers.forEach(gatherer => {
+            if (typeof gatherer.artifact === 'undefined') {
+              throw new Error(`${gatherer.constructor.name} failed to provide an artifact.`);
+            }
+
             artifacts[gatherer.name] = gatherer.artifact;
           });
         });
         return artifacts;
       });
   }
 
-  static getGathererClass(gatherer) {
-    return require(`./gatherers/${gatherer}`);
+  static getGathererClass(gatherer, paths) {
+    const rootPath = path.join(__dirname, '../../');
+
+    // Check each path to see if the gatherer can be located. First match wins.
+    const gathererDefinition = paths.reduce((definition, gathererPath) => {
+      // If the definition has already been found, just propagate it. Otherwise try a search
+      // on the path in this iteration of the loop.
+      if (definition !== null) {
+        return definition;
+      }
+
+      const requirePath = gathererPath.startsWith('/') ?
+          gathererPath :
+          path.join(rootPath, gathererPath);
+
+      try {
+        return require(`${requirePath}/${gatherer}`);
+      } catch (requireError) {
+        return null;
+      }
+    }, null);
+
+    if (!gathererDefinition) {
+      throw new Error(`Unable to locate gatherer: ${gatherer}`);
+    }
+
+    return gathererDefinition;
   }
 
-  static instantiateGatherers(passes) {
+  static instantiateGatherers(passes, paths) {
     return passes.map(pass => {
       pass.gatherers = pass.gatherers.map(gatherer => {
         // If this is already instantiated, don't do anything else.
         if (typeof gatherer !== 'string') {
           return gatherer;
         }
 
-        const GathererClass = GatherRunner.getGathererClass(gatherer);
+        const GathererClass = GatherRunner.getGathererClass(gatherer, paths);
         return new GathererClass();
       });
 

lighthouse-core/test/audits/audit.js

@@ -26,6 +26,8 @@ class B extends Audit {
   static get meta() {
     return {};
   }
+
+  static audit() {}
 }
 
 describe('Audit', () => {
@@ -37,6 +39,14 @@ describe('Audit', () => {
     assert.doesNotThrow(_ => B.meta);
   });
 
+  it('throws if an audit does not override audit()', () => {
+    assert.throws(_ => A.audit());
+  });
+
+  it('does not throw if an audit overrides audit()', () => {
+    assert.doesNotThrow(_ => B.audit());
+  });
+
   it('throws if an audit does generate a result with a value', () => {
     assert.throws(_ => A.generateAuditResult({}));
   });

lighthouse-core/test/config/index.js

@@ -127,6 +127,39 @@ describe('Config', () => {
     return assert.equal(typeof config.audits[0], 'function');
   });
 
+  it('tests multiple paths for expanding audits', () => {
+    const config = new Config({
+      paths: {
+        audits: ['/fake-path/']
+      },
+      audits: ['user-timings']
+    });
+
+    assert.ok(Array.isArray(config.audits));
+    assert.equal(config.audits.length, 1);
+
+    return assert.throws(_ => new Config({
+      paths: {
+        audits: ['/fake-path/']
+      },
+      audits: ['non-existent-audit']
+    }));
+  });
+
+  it('throws when it finds invalid audits', () => {
+    const paths = {
+      audits: ['lighthouse-core/test/fixtures/invalid-audits']
+    };
+
+    assert.throws(_ => new Config({paths, audits: ['missing-meta']}));
+    assert.throws(_ => new Config({paths, audits: ['missing-audit']}));
+    assert.throws(_ => new Config({paths, audits: ['missing-category']}));
+    assert.throws(_ => new Config({paths, audits: ['missing-name']}));
+    assert.throws(_ => new Config({paths, audits: ['missing-description']}));
+    assert.throws(_ => new Config({paths, audits: ['missing-required-artifact']}));
+    return assert.throws(_ => new Config({paths, audits: ['missing-generate-audit-result']}));
+  });
+
   it('expands artifacts', () => {
     const config = new Config({
       artifacts: {