Spike AXE A11Y testing in Cypress

cypress/e2e/1-register/register.spec.ts

@@ -33,9 +33,12 @@ describe("Registration", () => {
     });
 
     it("registers an account and lands on the home screen", () => {
+        cy.injectAxe();
+
         cy.get(".mx_ServerPicker_change", { timeout: 15000 }).click();
         cy.get(".mx_ServerPickerDialog_continue").should("be.visible");
         cy.percySnapshot("Server Picker");
+        cy.checkA11y();
 
         cy.get(".mx_ServerPickerDialog_otherHomeserver").type(synapse.baseUrl);
         cy.get(".mx_ServerPickerDialog_continue").click();
@@ -46,6 +49,7 @@ describe("Registration", () => {
         // Hide the server text as it contains the randomly allocated Synapse port
         const percyCSS = ".mx_ServerPicker_server { visibility: hidden !important; }";
         cy.percySnapshot("Registration", { percyCSS });
+        cy.checkA11y();
 
         cy.get("#mx_RegistrationForm_username").type("alice");
         cy.get("#mx_RegistrationForm_password").type("totally a great password");
@@ -55,16 +59,21 @@ describe("Registration", () => {
 
         cy.get(".mx_RegistrationEmailPromptDialog").should("be.visible");
         cy.percySnapshot("Registration email prompt", { percyCSS });
+        cy.checkA11y();
         cy.get(".mx_RegistrationEmailPromptDialog button.mx_Dialog_primary").click();
 
         cy.stopMeasuring("create-account");
         cy.get(".mx_InteractiveAuthEntryComponents_termsPolicy").should("be.visible");
         cy.percySnapshot("Registration terms prompt", { percyCSS });
+        cy.checkA11y();
 
         cy.get(".mx_InteractiveAuthEntryComponents_termsPolicy input").click();
         cy.startMeasuring("from-submit-to-home");
         cy.get(".mx_InteractiveAuthEntryComponents_termsSubmit").click();
 
+        cy.get(".mx_UseCaseSelection_skip").should("exist");
+        cy.percySnapshot("Use-case selection screen");
+        cy.checkA11y();
         cy.get(".mx_UseCaseSelection_skip .mx_AccessibleButton").click();
 
         cy.url().should('contain', '/#/home');

cypress/e2e/14-timeline/timeline.spec.ts

@@ -20,7 +20,6 @@ import { MessageEvent } from "matrix-events-sdk";
 
 import type { ISendEventResponse } from "matrix-js-sdk/src/@types/requests";
 import type { EventType } from "matrix-js-sdk/src/@types/event";
-import type { MatrixClient } from "matrix-js-sdk/src/client";
 import { SynapseInstance } from "../../plugins/synapsedocker";
 import { SettingLevel } from "../../../src/settings/SettingLevel";
 import { Layout } from "../../../src/settings/enums/Layout";
@@ -46,10 +45,14 @@ const expectDisplayName = (e: JQuery<HTMLElement>, displayName: string): void =>
 };
 
 const expectAvatar = (e: JQuery<HTMLElement>, avatarUrl: string): void => {
-    cy.getClient().then((cli: MatrixClient) => {
+    cy.all([
+        cy.window({ log: false }),
+        cy.getClient(),
+    ]).then(([win, cli]) => {
+        const size = AVATAR_SIZE * win.devicePixelRatio;
         expect(e.find(".mx_BaseAvatar_image").attr("src")).to.equal(
             // eslint-disable-next-line no-restricted-properties
-            cli.mxcUrlToHttp(avatarUrl, AVATAR_SIZE, AVATAR_SIZE, AVATAR_RESIZE_METHOD),
+            cli.mxcUrlToHttp(avatarUrl, size, size, AVATAR_RESIZE_METHOD),
         );
     });
 };
@@ -75,15 +78,17 @@ describe("Timeline", () => {
         cy.startSynapse("default").then(data => {
             synapse = data;
             cy.initTestUser(synapse, OLD_NAME).then(() =>
-                cy.window({ log: false }).then(() => {
-                    cy.createRoom({ name: ROOM_NAME }).then(_room1Id => {
-                        roomId = _room1Id;
-                    });
+                cy.createRoom({ name: ROOM_NAME }).then(_room1Id => {
+                    roomId = _room1Id;
                 }),
             );
         });
     });
 
+    afterEach(() => {
+        cy.stopSynapse(synapse);
+    });
+
     describe("useOnlyCurrentProfiles", () => {
         beforeEach(() => {
             cy.uploadContent(OLD_AVATAR).then((url) => {
@@ -95,10 +100,6 @@ describe("Timeline", () => {
             });
         });
 
-        afterEach(() => {
-            cy.stopSynapse(synapse);
-        });
-
         it("should show historical profiles if disabled", () => {
             cy.setSettingValue("useOnlyCurrentProfiles", null, SettingLevel.ACCOUNT, false);
             sendEvent(roomId);
@@ -146,11 +147,16 @@ describe("Timeline", () => {
     });
 
     describe("message displaying", () => {
+        beforeEach(() => {
+            cy.injectAxe();
+        });
+
         it("should create and configure a room on IRC layout", () => {
             cy.visit("/#/room/" + roomId);
             cy.setSettingValue("layout", null, SettingLevel.DEVICE, Layout.IRC);
             cy.contains(".mx_RoomView_body .mx_GenericEventListSummary[data-layout=irc] " +
                 ".mx_GenericEventListSummary_summary", "created and configured the room.");
+            cy.get(".mx_Spinner").should("not.exist");
             cy.percySnapshot("Configured room on IRC layout");
         });
 
@@ -174,10 +180,12 @@ describe("Timeline", () => {
                 .should('have.css', "margin-inline-start", "104px")
                 .should('have.css', "inset-inline-start", "0px");
 
+            cy.get(".mx_Spinner").should("not.exist");
             // Exclude timestamp from snapshot
             const percyCSS = ".mx_RoomView_body .mx_EventTile_info .mx_MessageTimestamp "
                 + "{ visibility: hidden !important; }";
             cy.percySnapshot("Event line with inline start margin on IRC layout", { percyCSS });
+            cy.checkA11y();
         });
 
         it("should click top left of view source event toggle", () => {

cypress/e2e/2-login/login.spec.ts

@@ -41,8 +41,11 @@ describe("Login", () => {
         });
 
         it("logs in with an existing account and lands on the home screen", () => {
+            cy.injectAxe();
+
             cy.get("#mx_LoginForm_username", { timeout: 15000 }).should("be.visible");
             cy.percySnapshot("Login");
+            cy.checkA11y();
 
             cy.get(".mx_ServerPicker_change").click();
             cy.get(".mx_ServerPickerDialog_otherHomeserver").type(synapse.baseUrl);

cypress/plugins/index.ts

@@ -22,6 +22,7 @@ import { performance } from "./performance";
 import { synapseDocker } from "./synapsedocker";
 import { webserver } from "./webserver";
 import { docker } from "./docker";
+import { log } from "./log";
 
 /**
  * @type {Cypress.PluginConfig}
@@ -31,4 +32,5 @@ export default function(on: PluginEvents, config: PluginConfigOptions) {
     performance(on, config);
     synapseDocker(on, config);
     webserver(on, config);
+    log(on, config);
 }

cypress/plugins/log.ts

@@ -0,0 +1,35 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/// <reference types="cypress" />
+
+import PluginEvents = Cypress.PluginEvents;
+import PluginConfigOptions = Cypress.PluginConfigOptions;
+
+export function log(on: PluginEvents, config: PluginConfigOptions) {
+    on("task", {
+        log(message: string) {
+            console.log(message);
+
+            return null;
+        },
+        table(message: string) {
+            console.table(message);
+
+            return null;
+        },
+    });
+}

cypress/support/axe.ts

@@ -0,0 +1,61 @@
+/*
+Copyright 2022 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/// <reference types="cypress" />
+
+import "cypress-axe";
+import * as axe from "axe-core";
+import { Options } from "cypress-axe";
+
+import Chainable = Cypress.Chainable;
+
+function terminalLog(violations: axe.Result[]): void {
+    cy.task(
+        'log',
+        `${violations.length} accessibility violation${
+            violations.length === 1 ? '' : 's'
+        } ${violations.length === 1 ? 'was' : 'were'} detected`,
+    );
+
+    // pluck specific keys to keep the table readable
+    const violationData = violations.map(({ id, impact, description, nodes }) => ({
+        id,
+        impact,
+        description,
+        nodes: nodes.length,
+    }));
+
+    cy.task('table', violationData);
+}
+
+Cypress.Commands.overwrite("checkA11y", (
+    originalFn: Chainable["checkA11y"],
+    context?: string | Node | axe.ContextObject | undefined,
+    options: Options = {},
+    violationCallback?: ((violations: axe.Result[]) => void) | undefined,
+    skipFailures?: boolean,
+): void => {
+    return originalFn(context, {
+        ...options,
+        rules: {
+            // Disable contrast checking for now as we have too many issues with it
+            'color-contrast': {
+                enabled: false,
+            },
+            ...options.rules,
+        },
+    }, violationCallback ?? terminalLog, skipFailures);
+});

cypress/support/e2e.ts

@@ -36,3 +36,4 @@ import "./iframes";
 import "./timeline";
 import "./network";
 import "./composer";
+import "./axe";

cypress/tsconfig.json

@@ -7,7 +7,11 @@
             "dom",
             "dom.iterable"
         ],
-        "types": ["cypress", "@percy/cypress"],
+        "types": [
+            "cypress",
+            "cypress-axe",
+            "@percy/cypress"
+        ],
         "resolveJsonModule": true,
         "esModuleInterop": true,
         "moduleResolution": "node",

package.json

@@ -168,10 +168,12 @@
     "@typescript-eslint/parser": "^5.6.0",
     "@wojtekmaj/enzyme-adapter-react-17": "^0.6.1",
     "allchange": "^1.0.6",
+    "axe-core": "^4.4.3",
     "babel-jest": "^26.6.3",
     "blob-polyfill": "^6.0.20211015",
     "chokidar": "^3.5.1",
     "cypress": "^10.3.0",
+    "cypress-axe": "^1.0.0",
     "cypress-real-events": "^1.7.1",
     "enzyme": "^3.11.0",
     "enzyme-to-json": "^3.6.2",

res/css/views/auth/_AuthBody.pcss

@@ -29,20 +29,20 @@ limitations under the License.
         flex-direction: column;
     }
 
-    h2 {
+    h1 {
         font-size: $font-24px;
-        font-weight: 600;
+        font-weight: $font-semi-bold;
         margin-top: 8px;
         color: $authpage-primary-color;
     }
 
-    h3 {
+    h2 {
         font-size: $font-14px;
-        font-weight: 600;
+        font-weight: $font-semi-bold;
         color: $authpage-secondary-color;
     }
 
-    h3.mx_AuthBody_centered {
+    h2.mx_AuthBody_centered {
         text-align: center;
     }
 

res/css/views/dialogs/_ServerPickerDialog.pcss

@@ -35,11 +35,11 @@ limitations under the License.
             }
         }
 
-        > h4 {
+        > h2 {
             font-size: $font-15px;
             font-weight: $font-semi-bold;
             color: $secondary-content;
-            margin-left: 8px;
+            margin: 16px 0 16px 8px;
         }
 
         > a {

res/css/views/elements/_ServerPicker.pcss

@@ -24,7 +24,7 @@ limitations under the License.
     font-size: $font-14px;
     line-height: $font-20px;
 
-    > h3 {
+    > h2 {
         font-weight: $font-semi-bold;
         margin: 0 0 20px;
         grid-column: 1;

res/css/views/elements/_UseCaseSelection.pcss

@@ -45,7 +45,7 @@ limitations under the License.
             text-align: center;
         }
 
-        h4 {
+        h3 {
             margin: 0;
             font-weight: 400;
             font-size: $font-16px;

res/css/views/messages/_DateSeparator.pcss

@@ -30,9 +30,12 @@ limitations under the License.
     border-bottom: 1px solid $menu-selected-color;
 }
 
-.mx_DateSeparator > div {
+.mx_DateSeparator > h2 {
     margin: 0 25px;
     flex: 0 0 auto;
+    font-size: inherit;
+    font-weight: inherit;
+    color: inherit;
 }
 
 .mx_DateSeparator_jumpToDateMenu {

src/components/structures/LeftPanel.tsx

@@ -382,7 +382,7 @@ export default class LeftPanel extends React.Component<IProps, IState> {
 
         return (
             <div className={containerClasses}>
-                <aside className="mx_LeftPanel_roomListContainer">
+                <div className="mx_LeftPanel_roomListContainer">
                     { this.renderSearchDialExplore() }
                     { this.renderBreadcrumbs() }
                     { !this.props.isMinimized && (
@@ -401,7 +401,7 @@ export default class LeftPanel extends React.Component<IProps, IState> {
                             { roomList }
                         </div>
                     </div>
-                </aside>
+                </div>
             </div>
         );
     }

src/components/structures/LoggedInView.tsx

@@ -674,7 +674,7 @@ class LoggedInView extends React.Component<IProps, IState> {
                     <div className={bodyClasses}>
                         <div className='mx_LeftPanel_outerWrapper'>
                             <LeftPanelLiveShareWarning isMinimized={this.props.collapseLhs || false} />
-                            <div className='mx_LeftPanel_wrapper'>
+                            <nav className='mx_LeftPanel_wrapper'>
                                 <BackdropPanel
                                     blurMultiplier={0.5}
                                     backgroundImage={this.state.backgroundImage}
@@ -693,7 +693,7 @@ class LoggedInView extends React.Component<IProps, IState> {
                                         resizeNotifier={this.props.resizeNotifier}
                                     />
                                 </div>
-                            </div>
+                            </nav>
                         </div>
                         <ResizeHandle passRef={this.resizeHandler} id="lp-resizer" />
                         <div className="mx_RoomView_wrapper">