From 09ff7446adf5adf122062c5520c76ab2041fe10e Mon Sep 17 00:00:00 2001
From: Rob Dodson <robdodson@google.com>
Date: Wed, 13 Dec 2017 15:03:38 -0800
Subject: [PATCH 1/9] wip not applicable reporting

---
 .../audits/accessibility/axe-audit.js         | 23 ++++++++++++
 lighthouse-core/audits/audit.js               |  1 +
 .../report/v2/renderer/category-renderer.js   | 35 +++++++++++++++++--
 lighthouse-core/report/v2/report-generator.js | 35 +++++++++++++++++++
 4 files changed, 92 insertions(+), 2 deletions(-)

diff --git a/lighthouse-core/audits/accessibility/axe-audit.js b/lighthouse-core/audits/accessibility/axe-audit.js
index 3b1bf97dd032..98ee09e47462 100644
--- a/lighthouse-core/audits/accessibility/axe-audit.js
+++ b/lighthouse-core/audits/accessibility/axe-audit.js
@@ -19,6 +19,29 @@ class AxeAudit extends Audit {
    * @return {!AuditResult}
    */
   static audit(artifacts) {
+    // Indicate if a test is not applicable.
+    // This means aXe did not find any nodes which matched these checks.
+    // Note in Lighthouse we use the phrasing "Not Applicable" (aXe uses "inapplicable", which sounds weird).
+    const notApplicables = artifacts.Accessibility.inapplicable;
+    if (notApplicables && notApplicables.length) {
+      const isNotApplicable = notApplicables.find(result => result.id === this.meta.name);
+      if (isNotApplicable) {
+        return {
+          rawValue: false,
+          notApplicable: true,
+          extendedInfo: {},
+          details: {
+            type: 'list',
+            header: {
+              type: 'text',
+              text: 'View failing elements',
+            },
+            items: [],
+          },
+        };
+      }
+    }
+
     const violations = artifacts.Accessibility.violations;
     const rule = violations.find(result => result.id === this.meta.name);
 
diff --git a/lighthouse-core/audits/audit.js b/lighthouse-core/audits/audit.js
index 455111ec21d4..54ed5eb7043d 100644
--- a/lighthouse-core/audits/audit.js
+++ b/lighthouse-core/audits/audit.js
@@ -156,6 +156,7 @@ class Audit {
       scoringMode: audit.meta.scoringMode || Audit.SCORING_MODES.BINARY,
       informative: audit.meta.informative,
       manual: audit.meta.manual,
+      notApplicable: result.notApplicable,
       name: audit.meta.name,
       description: auditDescription,
       helpText: audit.meta.helpText,
diff --git a/lighthouse-core/report/v2/renderer/category-renderer.js b/lighthouse-core/report/v2/renderer/category-renderer.js
index 267200118c1b..f127a6ab8c80 100644
--- a/lighthouse-core/report/v2/renderer/category-renderer.js
+++ b/lighthouse-core/report/v2/renderer/category-renderer.js
@@ -238,6 +238,19 @@ class CategoryRenderer {
     return passedElem;
   }
 
+  /**
+   * @param {!Array<!Element>} elements
+   * @return {!Element}
+   */
+  _renderNotApplicableAuditsSection(elements) {
+    const notApplicableElem = this._renderAuditGroup({
+      title: `${elements.length} Not Applicable Audits`,
+    }, {expandable: true});
+    notApplicableElem.classList.add('lh-passed-audits');
+    elements.forEach(elem => notApplicableElem.appendChild(elem));
+    return notApplicableElem;
+  }
+
   /**
    * @param {!Array<!ReportRenderer.AuditJSON>} manualAudits
    * @param {!Object<string, !ReportRenderer.GroupJSON>} groupDefinitions
@@ -443,13 +456,18 @@ class CategoryRenderer {
     element.appendChild(this._renderCategoryScore(category));
 
     const manualAudits = category.audits.filter(audit => audit.result.manual);
-    const nonManualAudits = category.audits.filter(audit => !manualAudits.includes(audit));
+    const nonManualAudits = category.audits.filter(audit => !manualAudits.includes(audit) && !audit.notApplicable);
     const auditsGroupedByGroup = /** @type {!Object<string,
         {passed: !Array<!ReportRenderer.AuditJSON>,
         failed: !Array<!ReportRenderer.AuditJSON>}>} */ ({});
     nonManualAudits.forEach(audit => {
       const groupId = audit.group;
-      const groups = auditsGroupedByGroup[groupId] || {passed: [], failed: []};
+      const groups = auditsGroupedByGroup[groupId] || {passed: [], failed: [], notApplicable: []};
+
+      if (audit.notApplicable) {
+        groups.notApplicable.push(audit);
+        return;
+      }
 
       if (audit.score === 100) {
         groups.passed.push(audit);
@@ -461,6 +479,7 @@ class CategoryRenderer {
     });
 
     const passedElements = /** @type {!Array<!Element>} */ ([]);
+    const notApplicableElements = /** @type {!Array<!Element>} */ ([]);
     Object.keys(auditsGroupedByGroup).forEach(groupId => {
       const group = groupDefinitions[groupId];
       const groups = auditsGroupedByGroup[groupId];
@@ -476,6 +495,12 @@ class CategoryRenderer {
         groups.passed.forEach(item => auditGroupElem.appendChild(this._renderAudit(item)));
         passedElements.push(auditGroupElem);
       }
+
+      if (groups.notApplicable.length) {
+        const auditGroupElem = this._renderAuditGroup(group, {expandable: true});
+        groups.notApplicable.forEach(item => auditGroupElem.appendChild(this._renderAudit(item)));
+        notApplicableElements.push(auditGroupElem);
+      }
     });
 
     // don't create a passed section if there are no passed
@@ -484,6 +509,12 @@ class CategoryRenderer {
     const passedElem = this._renderPassedAuditsSection(passedElements);
     element.appendChild(passedElem);
 
+    // don't create a not applicable section if there are no not applicables
+    if (!notApplicableElements.length) return element;
+
+    const notApplicableElem = this._renderNotApplicableAuditsSection(notApplicableElements);
+    element.appendChild(notApplicableElem);
+
     // Render manual audits after passing.
     this._renderManualAudits(manualAudits, groupDefinitions, element);
 
diff --git a/lighthouse-core/report/v2/report-generator.js b/lighthouse-core/report/v2/report-generator.js
index b16e5839a66f..58d20434d3bd 100644
--- a/lighthouse-core/report/v2/report-generator.js
+++ b/lighthouse-core/report/v2/report-generator.js
@@ -64,6 +64,41 @@ class ReportGeneratorV2 {
         .join(firstReplacement.replacement);
   }
 
+<<<<<<< HEAD
+=======
+  /**
+   * Returns the report JSON object with computed scores.
+   * @param {{categories: !Object<string, {id: string|undefined, weight: number|undefined, audits: !Array<{id: string, weight: number|undefined}>}>}} config
+   * @param {!Object<{score: ?number|boolean|undefined}>} resultsByAuditId
+   * @return {{score: number, categories: !Array<{audits: !Array<{score: number, result: !Object}>}>}}
+   */
+  generateReportJson(config, resultsByAuditId) {
+    const categories = Object.keys(config.categories).map(categoryId => {
+      const category = config.categories[categoryId];
+      category.id = categoryId;
+
+      const audits = category.audits.filter(audit => {
+        const result = resultsByAuditId[audit.id];
+        return !result.notApplicable;
+      }).map(audit => {
+        const result = resultsByAuditId[audit.id];
+        // Cast to number to catch `null` and undefined when audits error
+        let auditScore = Number(result.score) || 0;
+        if (typeof result.score === 'boolean') {
+          auditScore = result.score ? 100 : 0;
+        }
+
+        return Object.assign({}, audit, {result, score: auditScore});
+      });
+
+      const categoryScore = ReportGeneratorV2.arithmeticMean(audits);
+      return Object.assign({}, category, {audits, score: categoryScore});
+    });
+
+    const overallScore = ReportGeneratorV2.arithmeticMean(categories);
+    return {score: overallScore, categories};
+  }
+>>>>>>> wip not applicable reporting
 
   /**
    * Returns the report HTML as a string with the report JSON and renderer JS inlined.

From 52345323121ceb6c3eea4131bbd9845f82b4e899 Mon Sep 17 00:00:00 2001
From: Rob Dodson <robdodson@google.com>
Date: Wed, 13 Dec 2017 18:09:46 -0800
Subject: [PATCH 2/9] Add non-applicable test reporting for a11y

---
 .../audits/accessibility/axe-audit.js         | 34 +++++++--------
 .../gather/gatherers/accessibility.js         |  2 +-
 .../report/v2/renderer/category-renderer.js   | 42 ++++++++++++++-----
 lighthouse-core/report/v2/report-generator.js | 35 ++++++++++++++--
 4 files changed, 79 insertions(+), 34 deletions(-)

diff --git a/lighthouse-core/audits/accessibility/axe-audit.js b/lighthouse-core/audits/accessibility/axe-audit.js
index 98ee09e47462..42eafc67bea3 100644
--- a/lighthouse-core/audits/accessibility/axe-audit.js
+++ b/lighthouse-core/audits/accessibility/axe-audit.js
@@ -22,27 +22,25 @@ class AxeAudit extends Audit {
     // Indicate if a test is not applicable.
     // This means aXe did not find any nodes which matched these checks.
     // Note in Lighthouse we use the phrasing "Not Applicable" (aXe uses "inapplicable", which sounds weird).
-    const notApplicables = artifacts.Accessibility.inapplicable;
-    if (notApplicables && notApplicables.length) {
-      const isNotApplicable = notApplicables.find(result => result.id === this.meta.name);
-      if (isNotApplicable) {
-        return {
-          rawValue: false,
-          notApplicable: true,
-          extendedInfo: {},
-          details: {
-            type: 'list',
-            header: {
-              type: 'text',
-              text: 'View failing elements',
-            },
-            items: [],
+    const notApplicables = artifacts.Accessibility.notApplicable || [];
+    const isNotApplicable = notApplicables.find(result => result.id === this.meta.name);
+    if (isNotApplicable) {
+      return {
+        rawValue: false,
+        notApplicable: true,
+        extendedInfo: {},
+        details: {
+          type: 'list',
+          header: {
+            type: 'text',
+            text: 'View failing elements',
           },
-        };
-      }
+          items: [],
+        },
+      };
     }
 
-    const violations = artifacts.Accessibility.violations;
+    const violations = artifacts.Accessibility.violations || [];
     const rule = violations.find(result => result.id === this.meta.name);
 
     let nodeDetails = [];
diff --git a/lighthouse-core/gather/gatherers/accessibility.js b/lighthouse-core/gather/gatherers/accessibility.js
index 913cc43ed2f8..236b5242ea2b 100644
--- a/lighthouse-core/gather/gatherers/accessibility.js
+++ b/lighthouse-core/gather/gatherers/accessibility.js
@@ -43,7 +43,7 @@ function runA11yChecks() {
     }));
 
     // We only need violations, and circular references are possible outside of violations
-    axeResult = {violations: axeResult.violations};
+    axeResult = {violations: axeResult.violations, notApplicable: axeResult.inapplicable};
     return axeResult;
   });
 
diff --git a/lighthouse-core/report/v2/renderer/category-renderer.js b/lighthouse-core/report/v2/renderer/category-renderer.js
index f127a6ab8c80..ac3bef2c2751 100644
--- a/lighthouse-core/report/v2/renderer/category-renderer.js
+++ b/lighthouse-core/report/v2/renderer/category-renderer.js
@@ -225,13 +225,33 @@ class CategoryRenderer {
     return element;
   }
 
+  /**
+   * Find the total number of audits contained within a section.
+   * Accounts for nested subsections like Accessibility.
+   * @param {!Array<!Element>} elements
+   * @return {number}
+   */
+  _getTotalAuditsLength(elements) {
+    // Create a scratch element to append sections to so we can reuse querySelectorAll().
+    const scratch = this._dom.createElement('div');
+    elements.forEach(function(element) {
+      scratch.appendChild(element);
+    });
+    const subAudits = scratch.querySelectorAll('.lh-audit-group .lh-audit');
+    if (subAudits.length) {
+      return subAudits.length;
+    } else {
+      return elements.length;
+    }
+  }
+
   /**
    * @param {!Array<!Element>} elements
    * @return {!Element}
    */
   _renderPassedAuditsSection(elements) {
     const passedElem = this._renderAuditGroup({
-      title: `${elements.length} Passed Audits`,
+      title: `${this._getTotalAuditsLength(elements)} Passed Audits`,
     }, {expandable: true});
     passedElem.classList.add('lh-passed-audits');
     elements.forEach(elem => passedElem.appendChild(elem));
@@ -244,7 +264,7 @@ class CategoryRenderer {
    */
   _renderNotApplicableAuditsSection(elements) {
     const notApplicableElem = this._renderAuditGroup({
-      title: `${elements.length} Not Applicable Audits`,
+      title: `${this._getTotalAuditsLength(elements)} Not Applicable Audits`,
     }, {expandable: true});
     notApplicableElem.classList.add('lh-passed-audits');
     elements.forEach(elem => notApplicableElem.appendChild(elem));
@@ -456,23 +476,23 @@ class CategoryRenderer {
     element.appendChild(this._renderCategoryScore(category));
 
     const manualAudits = category.audits.filter(audit => audit.result.manual);
-    const nonManualAudits = category.audits.filter(audit => !manualAudits.includes(audit) && !audit.notApplicable);
+    const nonManualAudits = category.audits.filter(audit => !manualAudits.includes(audit));
     const auditsGroupedByGroup = /** @type {!Object<string,
         {passed: !Array<!ReportRenderer.AuditJSON>,
-        failed: !Array<!ReportRenderer.AuditJSON>}>} */ ({});
+        failed: !Array<!ReportRenderer.AuditJSON>,
+        notApplicable: !Array<!ReportRenderer.AuditJSON>}>} */ ({});
     nonManualAudits.forEach(audit => {
       const groupId = audit.group;
       const groups = auditsGroupedByGroup[groupId] || {passed: [], failed: [], notApplicable: []};
 
-      if (audit.notApplicable) {
+      if (audit.result.notApplicable) {
         groups.notApplicable.push(audit);
-        return;
-      }
-
-      if (audit.score === 100) {
-        groups.passed.push(audit);
       } else {
-        groups.failed.push(audit);
+        if (audit.score === 100) {
+          groups.passed.push(audit);
+        } else {
+          groups.failed.push(audit);
+        }
       }
 
       auditsGroupedByGroup[groupId] = groups;
diff --git a/lighthouse-core/report/v2/report-generator.js b/lighthouse-core/report/v2/report-generator.js
index 58d20434d3bd..a227eee746f7 100644
--- a/lighthouse-core/report/v2/report-generator.js
+++ b/lighthouse-core/report/v2/report-generator.js
@@ -44,6 +44,27 @@ class ReportGeneratorV2 {
     return REPORT_TEMPLATES;
   }
 
+<<<<<<< HEAD
+=======
+  /**
+   * Computes the weighted-average of the score of the list of items.
+   * @param {!Array<{score: number|undefined, weight: number|undefined}>} items
+   * @return {number}
+   */
+  static arithmeticMean(items) {
+    // debugger;
+    const results = items.reduce((result, item) => {
+      const score = Number(item.score) || 0;
+      const weight = Number(item.weight) || 0;
+      return {
+        weight: result.weight + weight,
+        sum: result.sum + score * weight,
+      };
+    }, {weight: 0, sum: 0});
+
+    return (results.sum / results.weight) || 0;
+  }
+>>>>>>> Add non-applicable test reporting for a11y
 
   /**
    * Replaces all the specified strings in source without serial replacements.
@@ -77,16 +98,22 @@ class ReportGeneratorV2 {
       const category = config.categories[categoryId];
       category.id = categoryId;
 
-      const audits = category.audits.filter(audit => {
-        const result = resultsByAuditId[audit.id];
-        return !result.notApplicable;
-      }).map(audit => {
+      const audits = category.audits.map(audit => {
         const result = resultsByAuditId[audit.id];
         // Cast to number to catch `null` and undefined when audits error
         let auditScore = Number(result.score) || 0;
         if (typeof result.score === 'boolean') {
           auditScore = result.score ? 100 : 0;
         }
+        // If a result was not applicable, meaning its checks did not run against anything on
+        // the page, force it's weight to 0. It will not count during the arithmeticMean() but
+        // will still be included in the final report json and displayed in the report as
+        // "Not Applicable".
+        if (result.notApplicable) {
+          audit.weight = 0;
+          result.informative = true;
+          auditScore = 100;
+        }
 
         return Object.assign({}, audit, {result, score: auditScore});
       });

From e70e6d3d72bc09d71398c1e496b7615cfe6fe7a7 Mon Sep 17 00:00:00 2001
From: Rob Dodson <robdodson@google.com>
Date: Wed, 13 Dec 2017 18:15:00 -0800
Subject: [PATCH 3/9] Fix closure issue

---
 lighthouse-core/report/v2/renderer/report-renderer.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lighthouse-core/report/v2/renderer/report-renderer.js b/lighthouse-core/report/v2/renderer/report-renderer.js
index 1744ffd84a2c..6df5e90c119d 100644
--- a/lighthouse-core/report/v2/renderer/report-renderer.js
+++ b/lighthouse-core/report/v2/renderer/report-renderer.js
@@ -183,6 +183,7 @@ if (typeof module !== 'undefined' && module.exports) {
  *       description: string,
  *       informative: boolean,
  *       manual: boolean,
+ *       notApplicable: boolean,
  *       debugString: string,
  *       displayValue: string,
  *       helpText: string,

From 514871fb99d66f3e7400f2a8623c69341838ff1e Mon Sep 17 00:00:00 2001
From: Rob Dodson <robdodson@google.com>
Date: Wed, 13 Dec 2017 18:23:36 -0800
Subject: [PATCH 4/9] Clean up unused details

---
 lighthouse-core/audits/accessibility/axe-audit.js | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/lighthouse-core/audits/accessibility/axe-audit.js b/lighthouse-core/audits/accessibility/axe-audit.js
index 42eafc67bea3..c160dd8df4e4 100644
--- a/lighthouse-core/audits/accessibility/axe-audit.js
+++ b/lighthouse-core/audits/accessibility/axe-audit.js
@@ -28,15 +28,6 @@ class AxeAudit extends Audit {
       return {
         rawValue: false,
         notApplicable: true,
-        extendedInfo: {},
-        details: {
-          type: 'list',
-          header: {
-            type: 'text',
-            text: 'View failing elements',
-          },
-          items: [],
-        },
       };
     }
 

From 91a48765ded586b6572a00290a0a920f19ae28ff Mon Sep 17 00:00:00 2001
From: Rob Dodson <robdodson@google.com>
Date: Thu, 21 Dec 2017 16:04:04 -0800
Subject: [PATCH 5/9] Tune up and add tests

---
 .../report/v2/renderer/category-renderer.js   | 30 ++++++++-----------
 lighthouse-core/report/v2/report-generator.js |  3 +-
 .../v2/renderer/category-renderer-test.js     | 21 ++++++++++---
 lighthouse-core/test/results/sample_v2.json   |  2 ++
 4 files changed, 33 insertions(+), 23 deletions(-)

diff --git a/lighthouse-core/report/v2/renderer/category-renderer.js b/lighthouse-core/report/v2/renderer/category-renderer.js
index ac3bef2c2751..dff47f1caf74 100644
--- a/lighthouse-core/report/v2/renderer/category-renderer.js
+++ b/lighthouse-core/report/v2/renderer/category-renderer.js
@@ -237,7 +237,7 @@ class CategoryRenderer {
     elements.forEach(function(element) {
       scratch.appendChild(element);
     });
-    const subAudits = scratch.querySelectorAll('.lh-audit-group .lh-audit');
+    const subAudits = scratch.querySelectorAll('.lh-audit');
     if (subAudits.length) {
       return subAudits.length;
     } else {
@@ -266,7 +266,7 @@ class CategoryRenderer {
     const notApplicableElem = this._renderAuditGroup({
       title: `${this._getTotalAuditsLength(elements)} Not Applicable Audits`,
     }, {expandable: true});
-    notApplicableElem.classList.add('lh-passed-audits');
+    notApplicableElem.classList.add('lh-audit-group--notapplicable');
     elements.forEach(elem => notApplicableElem.appendChild(elem));
     return notApplicableElem;
   }
@@ -487,12 +487,10 @@ class CategoryRenderer {
 
       if (audit.result.notApplicable) {
         groups.notApplicable.push(audit);
+      } else if (audit.score === 100) {
+        groups.passed.push(audit);
       } else {
-        if (audit.score === 100) {
-          groups.passed.push(audit);
-        } else {
-          groups.failed.push(audit);
-        }
+        groups.failed.push(audit);
       }
 
       auditsGroupedByGroup[groupId] = groups;
@@ -523,17 +521,15 @@ class CategoryRenderer {
       }
     });
 
-    // don't create a passed section if there are no passed
-    if (!passedElements.length) return element;
-
-    const passedElem = this._renderPassedAuditsSection(passedElements);
-    element.appendChild(passedElem);
-
-    // don't create a not applicable section if there are no not applicables
-    if (!notApplicableElements.length) return element;
+    if (passedElements.length) {
+      const passedElem = this._renderPassedAuditsSection(passedElements);
+      element.appendChild(passedElem);
+    }
 
-    const notApplicableElem = this._renderNotApplicableAuditsSection(notApplicableElements);
-    element.appendChild(notApplicableElem);
+    if (notApplicableElements.length) {
+      const notApplicableElem = this._renderNotApplicableAuditsSection(notApplicableElements);
+      element.appendChild(notApplicableElem);
+    }
 
     // Render manual audits after passing.
     this._renderManualAudits(manualAudits, groupDefinitions, element);
diff --git a/lighthouse-core/report/v2/report-generator.js b/lighthouse-core/report/v2/report-generator.js
index a227eee746f7..e2ef497f25b7 100644
--- a/lighthouse-core/report/v2/report-generator.js
+++ b/lighthouse-core/report/v2/report-generator.js
@@ -52,7 +52,6 @@ class ReportGeneratorV2 {
    * @return {number}
    */
   static arithmeticMean(items) {
-    // debugger;
     const results = items.reduce((result, item) => {
       const score = Number(item.score) || 0;
       const weight = Number(item.weight) || 0;
@@ -110,9 +109,9 @@ class ReportGeneratorV2 {
         // will still be included in the final report json and displayed in the report as
         // "Not Applicable".
         if (result.notApplicable) {
+          auditScore = 100;
           audit.weight = 0;
           result.informative = true;
-          auditScore = 100;
         }
 
         return Object.assign({}, audit, {result, score: auditScore});
diff --git a/lighthouse-core/test/report/v2/renderer/category-renderer-test.js b/lighthouse-core/test/report/v2/renderer/category-renderer-test.js
index 336f2e0af454..cade016e65b3 100644
--- a/lighthouse-core/test/report/v2/renderer/category-renderer-test.js
+++ b/lighthouse-core/test/report/v2/renderer/category-renderer-test.js
@@ -126,6 +126,18 @@ describe('CategoryRenderer', () => {
     assert.ok(!categoryDOM2.querySelector('.lh-audit-group--manual'));
   });
 
+  it('renders not applicable audits if the category contains them', () => {
+    const a11yCategory = sampleResults.reportCategories.find(cat => cat.id === 'accessibility');
+    const categoryDOM = renderer.render(a11yCategory, sampleResults.reportGroups);
+    assert.ok(categoryDOM.querySelector('.lh-audit-group--notapplicable .lh-audit-group__summary'));
+    assert.equal(categoryDOM.querySelectorAll('.lh-score--informative').length, 1,
+        'score shows informative and dash icon');
+
+    const perfCategory = sampleResults.reportCategories.find(cat => cat.id === 'performance');
+    const categoryDOM2 = renderer.render(perfCategory, sampleResults.reportGroups);
+    assert.ok(!categoryDOM2.querySelector('.lh-audit-group--notapplicable'));
+  });
+
   describe('performance category', () => {
     const category = sampleResults.reportCategories.find(cat => cat.id === 'performance');
 
@@ -245,12 +257,13 @@ describe('CategoryRenderer', () => {
 
     it('renders the failed audits grouped by group', () => {
       const categoryDOM = renderer.render(category, sampleResults.reportGroups);
-
-      const failedAudits = category.audits.filter(audit => audit.score !== 100);
+      const failedAudits = category.audits.filter(audit => {
+        return audit.score !== 100 && !audit.result.notApplicable;
+      });
       const failedAuditTags = new Set(failedAudits.map(audit => audit.group));
 
-      const failedAuditGroups = categoryDOM.querySelectorAll('.lh-category > .lh-audit-group');
-      assert.equal(failedAuditGroups.length, failedAuditTags.size+1);
+      const failedAuditGroups = categoryDOM.querySelectorAll('.lh-category > div.lh-audit-group');
+      assert.equal(failedAuditGroups.length, failedAuditTags.size);
     });
 
     it('renders the passed audits grouped by group', () => {
diff --git a/lighthouse-core/test/results/sample_v2.json b/lighthouse-core/test/results/sample_v2.json
index d217dc1c8429..c6fd5fe34553 100644
--- a/lighthouse-core/test/results/sample_v2.json
+++ b/lighthouse-core/test/results/sample_v2.json
@@ -4964,6 +4964,8 @@
             "score": true,
             "displayValue": "",
             "rawValue": true,
+            "notApplicable": true,
+            "informative": true,
             "extendedInfo": {},
             "scoringMode": "binary",
             "name": "accesskeys",

From d6a518ce9d9771ffba687f83c720a18fb9e28760 Mon Sep 17 00:00:00 2001
From: Rob Dodson <robdodson@google.com>
Date: Thu, 28 Dec 2017 12:06:56 -0800
Subject: [PATCH 6/9] Add additional info to hacking docs

---
 docs/hacking-tips.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/hacking-tips.md b/docs/hacking-tips.md
index 3d77ac1d6e15..2cce49e792a0 100644
--- a/docs/hacking-tips.md
+++ b/docs/hacking-tips.md
@@ -16,7 +16,7 @@ node --trace-warnings lighthouse-cli http://example.com
 ## Updating fixture dumps
 
 `lighthouse-core/test/results/samples_v2.json` is generated from running LH against
-dbw_tester.html. To update this file, run:
+dbw_tester.html. To update this file, start a local server on port `8080` and serve the directory `lighthouse-cli/test/fixtures`. Then run:
 
 ```sh
 npm run start -- --output=json --output-path=lighthouse-core/test/results/sample_v2.json http://localhost:8080/dobetterweb/dbw_tester.html

From 4f51b19b4acc6ea156f3c5da1a41850510309213 Mon Sep 17 00:00:00 2001
From: Rob Dodson <robdodson@google.com>
Date: Thu, 4 Jan 2018 13:54:50 -0800
Subject: [PATCH 7/9] Rev to latest axe-core

---
 package.json | 2 +-
 yarn.lock    | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/package.json b/package.json
index dfc2ea019ffc..b5530095c891 100644
--- a/package.json
+++ b/package.json
@@ -77,7 +77,7 @@
     "zone.js": "^0.7.3"
   },
   "dependencies": {
-    "axe-core": "2.4.1",
+    "axe-core": "2.6.1",
     "chrome-devtools-frontend": "1.0.422034",
     "chrome-launcher": "0.8.1",
     "configstore": "^3.1.1",
diff --git a/yarn.lock b/yarn.lock
index 4f13c9564ab8..26fe719e88c0 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -335,9 +335,9 @@ aws4@^1.2.1:
   version "1.4.1"
   resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.4.1.tgz#fde7d5292466d230e5ee0f4e038d9dfaab08fc61"
 
-axe-core@2.4.1:
-  version "2.4.1"
-  resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-2.4.1.tgz#55b6ceaa847cb1eaef5d559b6c41035dc3e07dbf"
+axe-core@2.6.1:
+  version "2.6.1"
+  resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-2.6.1.tgz#28772c4f76966d373acda35b9a409299dc00d1b5"
 
 axios@0.15.3:
   version "0.15.3"

From 871a17e0cdf78ea260acb9b37302b90cdb19ee27 Mon Sep 17 00:00:00 2001
From: Rob Dodson <robdodson@google.com>
Date: Thu, 4 Jan 2018 14:03:34 -0800
Subject: [PATCH 8/9] Move not applicable scoring over to new scoring.js module

---
 lighthouse-core/report/v2/report-generator.js | 61 -------------------
 lighthouse-core/scoring.js                    |  9 +++
 2 files changed, 9 insertions(+), 61 deletions(-)

diff --git a/lighthouse-core/report/v2/report-generator.js b/lighthouse-core/report/v2/report-generator.js
index e2ef497f25b7..b16e5839a66f 100644
--- a/lighthouse-core/report/v2/report-generator.js
+++ b/lighthouse-core/report/v2/report-generator.js
@@ -44,26 +44,6 @@ class ReportGeneratorV2 {
     return REPORT_TEMPLATES;
   }
 
-<<<<<<< HEAD
-=======
-  /**
-   * Computes the weighted-average of the score of the list of items.
-   * @param {!Array<{score: number|undefined, weight: number|undefined}>} items
-   * @return {number}
-   */
-  static arithmeticMean(items) {
-    const results = items.reduce((result, item) => {
-      const score = Number(item.score) || 0;
-      const weight = Number(item.weight) || 0;
-      return {
-        weight: result.weight + weight,
-        sum: result.sum + score * weight,
-      };
-    }, {weight: 0, sum: 0});
-
-    return (results.sum / results.weight) || 0;
-  }
->>>>>>> Add non-applicable test reporting for a11y
 
   /**
    * Replaces all the specified strings in source without serial replacements.
@@ -84,47 +64,6 @@ class ReportGeneratorV2 {
         .join(firstReplacement.replacement);
   }
 
-<<<<<<< HEAD
-=======
-  /**
-   * Returns the report JSON object with computed scores.
-   * @param {{categories: !Object<string, {id: string|undefined, weight: number|undefined, audits: !Array<{id: string, weight: number|undefined}>}>}} config
-   * @param {!Object<{score: ?number|boolean|undefined}>} resultsByAuditId
-   * @return {{score: number, categories: !Array<{audits: !Array<{score: number, result: !Object}>}>}}
-   */
-  generateReportJson(config, resultsByAuditId) {
-    const categories = Object.keys(config.categories).map(categoryId => {
-      const category = config.categories[categoryId];
-      category.id = categoryId;
-
-      const audits = category.audits.map(audit => {
-        const result = resultsByAuditId[audit.id];
-        // Cast to number to catch `null` and undefined when audits error
-        let auditScore = Number(result.score) || 0;
-        if (typeof result.score === 'boolean') {
-          auditScore = result.score ? 100 : 0;
-        }
-        // If a result was not applicable, meaning its checks did not run against anything on
-        // the page, force it's weight to 0. It will not count during the arithmeticMean() but
-        // will still be included in the final report json and displayed in the report as
-        // "Not Applicable".
-        if (result.notApplicable) {
-          auditScore = 100;
-          audit.weight = 0;
-          result.informative = true;
-        }
-
-        return Object.assign({}, audit, {result, score: auditScore});
-      });
-
-      const categoryScore = ReportGeneratorV2.arithmeticMean(audits);
-      return Object.assign({}, category, {audits, score: categoryScore});
-    });
-
-    const overallScore = ReportGeneratorV2.arithmeticMean(categories);
-    return {score: overallScore, categories};
-  }
->>>>>>> wip not applicable reporting
 
   /**
    * Returns the report HTML as a string with the report JSON and renderer JS inlined.
diff --git a/lighthouse-core/scoring.js b/lighthouse-core/scoring.js
index 10b225d9ea1b..84e80e34c00d 100644
--- a/lighthouse-core/scoring.js
+++ b/lighthouse-core/scoring.js
@@ -43,6 +43,15 @@ class ReportScoring {
         if (typeof result.score === 'boolean') {
           auditScore = result.score ? 100 : 0;
         }
+        // If a result was not applicable, meaning its checks did not run against anything on
+        // the page, force it's weight to 0. It will not count during the arithmeticMean() but
+        // will still be included in the final report json and displayed in the report as
+        // "Not Applicable".
+        if (result.notApplicable) {
+          auditScore = 100;
+          audit.weight = 0;
+          result.informative = true;
+        }
 
         return Object.assign({}, audit, {result, score: auditScore});
       });

From 450a33d6d491af9de69507ca68b58154f0a844cc Mon Sep 17 00:00:00 2001
From: Rob Dodson <robdodson@google.com>
Date: Thu, 4 Jan 2018 16:05:04 -0800
Subject: [PATCH 9/9] Fix busted smokehouse tests

---
 .../offline-local/offline-expectations.js      | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/lighthouse-cli/test/smokehouse/offline-local/offline-expectations.js b/lighthouse-cli/test/smokehouse/offline-local/offline-expectations.js
index a36ac695f656..7d5baa5f4292 100644
--- a/lighthouse-cli/test/smokehouse/offline-local/offline-expectations.js
+++ b/lighthouse-cli/test/smokehouse/offline-local/offline-expectations.js
@@ -51,22 +51,22 @@ module.exports = [
         score: false,
       },
       'aria-valid-attr': {
-        score: true,
+        notApplicable: true,
       },
       'aria-allowed-attr': {
-        score: true,
+        notApplicable: true,
       },
       'color-contrast': {
         score: true,
       },
       'image-alt': {
-        score: true,
+        notApplicable: true,
       },
       'label': {
-        score: true,
+        notApplicable: true,
       },
       'tabindex': {
-        score: true,
+        notApplicable: true,
       },
       'content-width': {
         score: true,
@@ -114,10 +114,10 @@ module.exports = [
         score: false,
       },
       'aria-valid-attr': {
-        score: true,
+        notApplicable: true,
       },
       'aria-allowed-attr': {
-        score: true,
+        notApplicable: true,
       },
       'color-contrast': {
         score: true,
@@ -126,10 +126,10 @@ module.exports = [
         score: false,
       },
       'label': {
-        score: true,
+        notApplicable: true,
       },
       'tabindex': {
-        score: true,
+        notApplicable: true,
       },
       'content-width': {
         score: true,