report(a11y): Don't count non-applicable a11y audits toward score

docs/hacking-tips.md

@@ -16,7 +16,7 @@ node --trace-warnings lighthouse-cli http://example.com
 ## Updating fixture dumps
 
 `lighthouse-core/test/results/samples_v2.json` is generated from running LH against
-dbw_tester.html. To update this file, run:
+dbw_tester.html. To update this file, start a local server on port `8080` and serve the directory `lighthouse-cli/test/fixtures`. Then run:
 
 ```sh
 npm run start -- --output=json --output-path=lighthouse-core/test/results/sample_v2.json http://localhost:8080/dobetterweb/dbw_tester.html

lighthouse-cli/test/smokehouse/offline-local/offline-expectations.js

@@ -51,22 +51,22 @@ module.exports = [
         score: false,
       },
       'aria-valid-attr': {
-        score: true,
+        notApplicable: true,
       },
       'aria-allowed-attr': {
-        score: true,
+        notApplicable: true,
       },
       'color-contrast': {
         score: true,
       },
       'image-alt': {
-        score: true,
+        notApplicable: true,
       },
       'label': {
-        score: true,
+        notApplicable: true,
       },
       'tabindex': {
-        score: true,
+        notApplicable: true,
       },
       'content-width': {
         score: true,
@@ -114,10 +114,10 @@ module.exports = [
         score: false,
       },
       'aria-valid-attr': {
-        score: true,
+        notApplicable: true,
       },
       'aria-allowed-attr': {
-        score: true,
+        notApplicable: true,
       },
       'color-contrast': {
         score: true,
@@ -126,10 +126,10 @@ module.exports = [
         score: false,
       },
       'label': {
-        score: true,
+        notApplicable: true,
       },
       'tabindex': {
-        score: true,
+        notApplicable: true,
       },
       'content-width': {
         score: true,

lighthouse-core/audits/accessibility/axe-audit.js

@@ -19,7 +19,19 @@ class AxeAudit extends Audit {
    * @return {!AuditResult}
    */
   static audit(artifacts) {
-    const violations = artifacts.Accessibility.violations;
+    // Indicate if a test is not applicable.
+    // This means aXe did not find any nodes which matched these checks.
+    // Note in Lighthouse we use the phrasing "Not Applicable" (aXe uses "inapplicable", which sounds weird).
+    const notApplicables = artifacts.Accessibility.notApplicable || [];
+    const isNotApplicable = notApplicables.find(result => result.id === this.meta.name);
+    if (isNotApplicable) {
+      return {
+        rawValue: false,
+        notApplicable: true,
+      };
+    }
+
+    const violations = artifacts.Accessibility.violations || [];
     const rule = violations.find(result => result.id === this.meta.name);
 
     let nodeDetails = [];

lighthouse-core/audits/audit.js

@@ -156,6 +156,7 @@ class Audit {
       scoringMode: audit.meta.scoringMode || Audit.SCORING_MODES.BINARY,
       informative: audit.meta.informative,
       manual: audit.meta.manual,
+      notApplicable: result.notApplicable,
       name: audit.meta.name,
       description: auditDescription,
       helpText: audit.meta.helpText,

lighthouse-core/gather/gatherers/accessibility.js

@@ -43,7 +43,7 @@ function runA11yChecks() {
     }));
 
     // We only need violations, and circular references are possible outside of violations
-    axeResult = {violations: axeResult.violations};
+    axeResult = {violations: axeResult.violations, notApplicable: axeResult.inapplicable};
     return axeResult;
   });
 

lighthouse-core/report/v2/renderer/category-renderer.js

@@ -225,19 +225,52 @@ class CategoryRenderer {
     return element;
   }
 
+  /**
+   * Find the total number of audits contained within a section.
+   * Accounts for nested subsections like Accessibility.
+   * @param {!Array<!Element>} elements
+   * @return {number}
+   */
+  _getTotalAuditsLength(elements) {
+    // Create a scratch element to append sections to so we can reuse querySelectorAll().
+    const scratch = this._dom.createElement('div');
+    elements.forEach(function(element) {
+      scratch.appendChild(element);
+    });
+    const subAudits = scratch.querySelectorAll('.lh-audit');
+    if (subAudits.length) {
+      return subAudits.length;
+    } else {
+      return elements.length;
+    }
+  }
+
   /**
    * @param {!Array<!Element>} elements
    * @return {!Element}
    */
   _renderPassedAuditsSection(elements) {
     const passedElem = this._renderAuditGroup({
-      title: `${elements.length} Passed Audits`,
+      title: `${this._getTotalAuditsLength(elements)} Passed Audits`,
     }, {expandable: true});
     passedElem.classList.add('lh-passed-audits');
     elements.forEach(elem => passedElem.appendChild(elem));
     return passedElem;
   }
 
+  /**
+   * @param {!Array<!Element>} elements
+   * @return {!Element}
+   */
+  _renderNotApplicableAuditsSection(elements) {
+    const notApplicableElem = this._renderAuditGroup({
+      title: `${this._getTotalAuditsLength(elements)} Not Applicable Audits`,
+    }, {expandable: true});
+    notApplicableElem.classList.add('lh-audit-group--notapplicable');
+    elements.forEach(elem => notApplicableElem.appendChild(elem));
+    return notApplicableElem;
+  }
+
   /**
    * @param {!Array<!ReportRenderer.AuditJSON>} manualAudits
    * @param {!Object<string, !ReportRenderer.GroupJSON>} groupDefinitions
@@ -446,12 +479,15 @@ class CategoryRenderer {
     const nonManualAudits = category.audits.filter(audit => !manualAudits.includes(audit));
     const auditsGroupedByGroup = /** @type {!Object<string,
         {passed: !Array<!ReportRenderer.AuditJSON>,
-        failed: !Array<!ReportRenderer.AuditJSON>}>} */ ({});
+        failed: !Array<!ReportRenderer.AuditJSON>,
+        notApplicable: !Array<!ReportRenderer.AuditJSON>}>} */ ({});
     nonManualAudits.forEach(audit => {
       const groupId = audit.group;
-      const groups = auditsGroupedByGroup[groupId] || {passed: [], failed: []};
+      const groups = auditsGroupedByGroup[groupId] || {passed: [], failed: [], notApplicable: []};
 
-      if (audit.score === 100) {
+      if (audit.result.notApplicable) {
+        groups.notApplicable.push(audit);
+      } else if (audit.score === 100) {
         groups.passed.push(audit);
       } else {
         groups.failed.push(audit);
@@ -461,6 +497,7 @@ class CategoryRenderer {
     });
 
     const passedElements = /** @type {!Array<!Element>} */ ([]);
+    const notApplicableElements = /** @type {!Array<!Element>} */ ([]);
     Object.keys(auditsGroupedByGroup).forEach(groupId => {
       const group = groupDefinitions[groupId];
       const groups = auditsGroupedByGroup[groupId];
@@ -476,13 +513,23 @@ class CategoryRenderer {
         groups.passed.forEach(item => auditGroupElem.appendChild(this._renderAudit(item)));
         passedElements.push(auditGroupElem);
       }
+
+      if (groups.notApplicable.length) {
+        const auditGroupElem = this._renderAuditGroup(group, {expandable: true});
+        groups.notApplicable.forEach(item => auditGroupElem.appendChild(this._renderAudit(item)));
+        notApplicableElements.push(auditGroupElem);
+      }
     });
 
-    // don't create a passed section if there are no passed
-    if (!passedElements.length) return element;
+    if (passedElements.length) {
+      const passedElem = this._renderPassedAuditsSection(passedElements);
+      element.appendChild(passedElem);
+    }
 
-    const passedElem = this._renderPassedAuditsSection(passedElements);
-    element.appendChild(passedElem);
+    if (notApplicableElements.length) {
+      const notApplicableElem = this._renderNotApplicableAuditsSection(notApplicableElements);
+      element.appendChild(notApplicableElem);
+    }
 
     // Render manual audits after passing.
     this._renderManualAudits(manualAudits, groupDefinitions, element);

lighthouse-core/report/v2/renderer/report-renderer.js

@@ -183,6 +183,7 @@ if (typeof module !== 'undefined' && module.exports) {
  *       description: string,
  *       informative: boolean,
  *       manual: boolean,
+ *       notApplicable: boolean,
  *       debugString: string,
  *       displayValue: string,
  *       helpText: string,

lighthouse-core/scoring.js

@@ -43,6 +43,15 @@ class ReportScoring {
         if (typeof result.score === 'boolean') {
           auditScore = result.score ? 100 : 0;
         }
+        // If a result was not applicable, meaning its checks did not run against anything on
+        // the page, force it's weight to 0. It will not count during the arithmeticMean() but
+        // will still be included in the final report json and displayed in the report as
+        // "Not Applicable".
+        if (result.notApplicable) {
+          auditScore = 100;
+          audit.weight = 0;
+          result.informative = true;
+        }
 
         return Object.assign({}, audit, {result, score: auditScore});
       });

lighthouse-core/test/report/v2/renderer/category-renderer-test.js

@@ -126,6 +126,18 @@ describe('CategoryRenderer', () => {
     assert.ok(!categoryDOM2.querySelector('.lh-audit-group--manual'));
   });
 
+  it('renders not applicable audits if the category contains them', () => {
+    const a11yCategory = sampleResults.reportCategories.find(cat => cat.id === 'accessibility');
+    const categoryDOM = renderer.render(a11yCategory, sampleResults.reportGroups);
+    assert.ok(categoryDOM.querySelector('.lh-audit-group--notapplicable .lh-audit-group__summary'));
+    assert.equal(categoryDOM.querySelectorAll('.lh-score--informative').length, 1,
+        'score shows informative and dash icon');
+
+    const perfCategory = sampleResults.reportCategories.find(cat => cat.id === 'performance');
+    const categoryDOM2 = renderer.render(perfCategory, sampleResults.reportGroups);
+    assert.ok(!categoryDOM2.querySelector('.lh-audit-group--notapplicable'));
+  });
+
   describe('performance category', () => {
     const category = sampleResults.reportCategories.find(cat => cat.id === 'performance');
 
@@ -245,12 +257,13 @@ describe('CategoryRenderer', () => {
 
     it('renders the failed audits grouped by group', () => {
       const categoryDOM = renderer.render(category, sampleResults.reportGroups);
-
-      const failedAudits = category.audits.filter(audit => audit.score !== 100);
+      const failedAudits = category.audits.filter(audit => {
+        return audit.score !== 100 && !audit.result.notApplicable;
+      });
       const failedAuditTags = new Set(failedAudits.map(audit => audit.group));
 
-      const failedAuditGroups = categoryDOM.querySelectorAll('.lh-category > .lh-audit-group');
-      assert.equal(failedAuditGroups.length, failedAuditTags.size+1);
+      const failedAuditGroups = categoryDOM.querySelectorAll('.lh-category > div.lh-audit-group');
+      assert.equal(failedAuditGroups.length, failedAuditTags.size);
     });
 
     it('renders the passed audits grouped by group', () => {

lighthouse-core/test/results/sample_v2.json

@@ -4964,6 +4964,8 @@
             "score": true,
             "displayValue": "",
             "rawValue": true,
+            "notApplicable": true,
+            "informative": true,
             "extendedInfo": {},
             "scoringMode": "binary",
             "name": "accesskeys",

package.json

@@ -77,7 +77,7 @@
     "zone.js": "^0.7.3"
   },
   "dependencies": {
-    "axe-core": "2.4.1",
+    "axe-core": "2.6.1",
     "chrome-devtools-frontend": "1.0.422034",
     "chrome-launcher": "0.8.1",
     "configstore": "^3.1.1",

yarn.lock

@@ -335,9 +335,9 @@ aws4@^1.2.1:
   version "1.4.1"
   resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.4.1.tgz#fde7d5292466d230e5ee0f4e038d9dfaab08fc61"
 
-axe-core@2.4.1:
-  version "2.4.1"
-  resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-2.4.1.tgz#55b6ceaa847cb1eaef5d559b6c41035dc3e07dbf"
+axe-core@2.6.1:
+  version "2.6.1"
+  resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-2.6.1.tgz#28772c4f76966d373acda35b9a409299dc00d1b5"
 
 axios@0.15.3:
   version "0.15.3"