Fix E2E Tests, add CI break on test failure, improve TreeDumpLibrary

.ado/Get-ShellScaling.ps1

@@ -0,0 +1,26 @@
+Add-Type @'
+  using System; 
+  using System.Runtime.InteropServices;
+  using System.Drawing;
+
+  public class DPI {  
+    [DllImport("gdi32.dll")]
+    static extern int GetDeviceCaps(IntPtr hdc, int nIndex);
+
+    public enum DeviceCap {
+      VERTRES = 10,
+      DESKTOPVERTRES = 117
+    } 
+
+    public static float scaling() {
+      Graphics g = Graphics.FromHwnd(IntPtr.Zero);
+      IntPtr desktop = g.GetHdc();
+      int LogicalScreenHeight = GetDeviceCaps(desktop, (int)DeviceCap.VERTRES);
+      int PhysicalScreenHeight = GetDeviceCaps(desktop, (int)DeviceCap.DESKTOPVERTRES);
+
+      return (float)PhysicalScreenHeight / (float)LogicalScreenHeight;
+    }
+  }
+'@ -ReferencedAssemblies 'System.Drawing.dll'
+
+[Math]::round([DPI]::scaling(), 2) * 100

.ado/SetupLocalDumps.cmd

@@ -0,0 +1,11 @@
+setlocal
+set CRASHDUMPS_FOLDER=%BUILD_ARTIFACTSTAGINGDIRECTORY%\ReactUWPTestAppTreeDump\CrashDumps
+reg query "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" /s
+reg add  "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps\ReactUWPTestApp.exe" /v DumpFolder /t REG_SZ /d %CRASHDUMPS_FOLDER%
+reg add "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps\ReactUWPTestApp.exe" /v DumpType /t REG_DWORD /d 2
+if not exist %CRASHDUMPS_FOLDER% ( 
+    md %CRASHDUMPS_FOLDER%
+) else (
+    del %CRASHDUMPS_FOLDER%\* /s /q
+)
+endlocal

.ado/templates/e2e-test-job.yml

@@ -23,6 +23,12 @@ jobs:
 
       - template: prepare-env.yml
 
+      - task: CmdLine@2
+        displayName: Set LocalDumps
+        inputs:
+          script: $(Build.SourcesDirectory)\.ado\SetupLocalDumps.cmd
+          workingDirectory: $(Build.SourcesDirectory)
+
       - task: NuGetCommand@2
         displayName: NuGet restore - ReactUWPTestApp
         inputs:
@@ -74,6 +80,11 @@ jobs:
           targetType: 'inline'
           script: 'if ((Get-Process React*) -eq $Null) { echo "TestApp is not running"; exit 1}'
 
+      - task: CmdLine@2
+        displayName: mkdir errorShots
+        inputs:
+          script: mkdir $(Build.SourcesDirectory)\packages\E2ETest\errorShots
+
       - task: CmdLine@2
         displayName: run e2etest
         inputs:
@@ -87,7 +98,28 @@ jobs:
           targetFolder: $(Build.StagingDirectory)/ReactUWPTestAppTreeDump
           contents: TreeDump\**
         condition: succeededOrFailed()
-
+
+      - task: CopyFiles@2
+        displayName: Copy tree dump screenshots
+        inputs:
+          sourceFolder: $(Build.SourcesDirectory)\packages\E2ETest\errorShots
+          targetFolder: $(Build.StagingDirectory)/ReactUWPTestAppTreeDump
+        condition: succeededOrFailed()
+
+      - task: PowerShell@2
+        displayName: "MSInfo32"
+        inputs:
+          targetType: "inline"
+          script: "Start-Process msinfo32 -ArgumentList '/report $(Build.StagingDirectory)\\ReactUWPTestAppTreeDump\\msinfo32.txt' -Wait"
+        condition: failed()
+
+      - task: PowerShell@2
+        displayName: "Get-ShellScaling"
+        inputs:
+          targetType: "inline"
+          script: ". $(Build.SourcesDirectory)\\.ado\\Get-ShellScaling.ps1 > $(Build.StagingDirectory)\\ReactUWPTestAppTreeDump\\scaleFactor.txt"
+        condition: failed()
+
       - task: PublishBuildArtifacts@1
         displayName: "Publish Artifact:ReactUWPTestAppTreeDump"
         inputs:

.gitignore

@@ -155,4 +155,5 @@ bld/
 # OpenCover
 *_coverage.xml
 
-tsdoc-metadata.json
+tsdoc-metadata.json
+*.dmp

change/react-native-windows-2020-04-14-05-43-39-jsonTreeDump.json

@@ -0,0 +1,8 @@
+{
+  "type": "prerelease",
+  "comment": "run_wdio.js will run the tests but also set the exit code to zero/non-zero on success/failure respectively. This is important to break the CI/PR build on test failures, which we weren't doing until now.",
+  "packageName": "react-native-windows",
+  "email": "asklar@winse.microsoft.com",
+  "dependentChangeType": "patch",
+  "date": "2020-04-14T12:43:39.263Z"
+}

packages/E2ETest/app/ControlStyleTestPage.tsx

@@ -3,7 +3,8 @@
  * Licensed under the MIT License.
  */
 
-import { Switch, CheckBox, TextInput, View, StyleSheet, Button } from 'react-native';
+ // Bug:4596 Switch
+import { /* Switch, */ CheckBox, TextInput, View, StyleSheet, Button } from 'react-native';
 import { DatePicker, Picker } from 'react-native-windows';
 import React, { useState } from 'react';
 import { SHOWBORDER_ON_CONTROLSTYLE, TREE_DUMP_RESULT } from './Consts';
@@ -56,7 +57,11 @@ export function ControlStyleTestPage() {
    return (
     <View>
       <View testID={'ControlStyleView'}>
-        <Switch style={showRoundBorder? styles.roundBorder :styles.regularBorder} thumbColor='blue'/>
+        {
+        /*
+        // Bug:4596 Switch
+        <Switch style={showRoundBorder? styles.roundBorder :styles.regularBorder} thumbColor='blue'/> 
+        */}
         <CheckBox style={showRoundBorder? styles.roundBorder :styles.regularBorder} />
         <TextInput style={showRoundBorder? styles.roundBorder :styles.regularBorder}
           placeholder='TextBox'

packages/E2ETest/app/LoginTestPage.tsx

@@ -4,29 +4,42 @@
  */
 
 import React, { useState } from 'react';
-import { Text, TextInput, View, StyleSheet, TouchableOpacity, Button } from 'react-native';
-import { USERNAME_ON_LOGIN, PASSWORD_ON_LOGIN, SUBMIT_ON_LOGIN, LOGINRESULT_ON_LOGIN, SHOWPASSWORD_ON_LOGIN } from './Consts';
+import {
+  Text,
+  TextInput,
+  View,
+  StyleSheet,
+  TouchableOpacity,
+  Button,
+} from 'react-native';
+import {
+  USERNAME_ON_LOGIN,
+  PASSWORD_ON_LOGIN,
+  SUBMIT_ON_LOGIN,
+  LOGINRESULT_ON_LOGIN,
+  SHOWPASSWORD_ON_LOGIN,
+} from './Consts';
 
 const styles = StyleSheet.create({
   container: {
-    padding: 20
+    padding: 20,
   },
   input: {
     height: 40,
     backgroundColor: 'rgba(225,225,225,0.2)',
     marginBottom: 10,
     padding: 10,
-    color: '#fff'
+    color: '#000000',
   },
   buttonContainer: {
     backgroundColor: '#2980b6',
-    paddingVertical: 15
+    paddingVertical: 15,
   },
   buttonText: {
     color: '#fff',
     textAlign: 'center',
-    fontWeight: '700'
-  }
+    fontWeight: '700',
+  },
 });
 
 export function LoginTestPage() {
@@ -36,9 +49,9 @@ export function LoginTestPage() {
   const [passwordHidden, setPasswordHidden] = useState(true);
 
   const onPressShowPassword = () => {
-      var previousState = passwordHidden;
-      setPasswordHidden(!previousState);
-   }
+    var previousState = passwordHidden;
+    setPasswordHidden(!previousState);
+  };
 
   const onPress = () => {
     if (userName === 'username' && password === 'password') {
@@ -49,29 +62,42 @@ export function LoginTestPage() {
   };
   return (
     <View>
-      <TextInput style={styles.input}
-        placeholder='Email or Mobile Num'
-        placeholderTextColor='rgba(225,225,225,0.7)'
+      <TextInput
+        style={styles.input}
+        placeholder="Email or Mobile Num"
+        placeholderTextColor="rgba(225,225,225,0.7)"
         testID={USERNAME_ON_LOGIN}
-        onChange={(text) => { setUserName(text.nativeEvent.text) }} />
+        onChange={text => {
+          setUserName(text.nativeEvent.text);
+        }}
+      />
 
-      <TextInput style={styles.input}
-        placeholder='Password'
+      <TextInput
+        style={styles.input}
+        placeholder="Password"
         testID={PASSWORD_ON_LOGIN}
-        placeholderTextColor='rgba(225,225,225,0.7)'
-        secureTextEntry = {passwordHidden}
-        onChange={(text) => { setPassword(text.nativeEvent.text) }} />
+        placeholderTextColor="rgba(225,225,225,0.7)"
+        secureTextEntry={passwordHidden}
+        onChange={text => {
+          setPassword(text.nativeEvent.text);
+        }}
+      />
 
-      <Button title= {passwordHidden?"Show Password":"Hide Password"} 
-        onPress={onPressShowPassword} 
-        testID={SHOWPASSWORD_ON_LOGIN}/>
+      <Button
+        title={passwordHidden ? 'Show Password' : 'Hide Password'}
+        onPress={onPressShowPassword}
+        testID={SHOWPASSWORD_ON_LOGIN}
+      />
 
-      <TouchableOpacity style={styles.buttonContainer}
+      <TouchableOpacity
+        style={styles.buttonContainer}
         testID={SUBMIT_ON_LOGIN}
-        onPress={onPress}>
+        onPress={onPress}
+      >
         <Text style={styles.buttonText}>LOGIN</Text>
       </TouchableOpacity>
 
       <Text testID={LOGINRESULT_ON_LOGIN}>{loginState}</Text>
-    </View >);
-}
+    </View>
+  );
+}

packages/E2ETest/package.json

@@ -13,7 +13,7 @@
     "prettier": "prettier --write --loglevel warn \"**/**/*.ts\"",
     "e2e": "react-native run-windows --no-launch --logging && npm run e2etest",
     "e2ebundle": "npm run bundle && react-native run-windows --no-launch --no-packager --bundle --logging && npm run e2etest",
-    "e2etest": "rimraf reports/* && npm run prettier && wdio",
+    "e2etest": "rimraf reports/* && npm run prettier && node run_wdio.js",
     "testspec": "npm run prettier && wdio --spec",
     "bundle": "just-scripts prepareBundle && react-native bundle --platform windows --entry-file dist/app/index.js --bundle-output windows/ReactUWPTestApp/Bundle/index.windows.bundle --assets-dest windows/ReactUWPTestApp/Bundle",
     "buildapp": "react-native run-windows --no-launch --no-packager --no-deploy",
@@ -22,6 +22,7 @@
     "deploybundleapp": "react-native run-windows --no-launch --no-packager --no-build --bundle"
   },
   "dependencies": {
+    "prompt-sync": "^4.2.0",
     "react": "16.9.0",
     "react-native": "0.61.5",
     "react-native-windows": "0.0.0-master.34",

packages/E2ETest/run_wdio.js

@@ -0,0 +1,119 @@
+const path = require('path');
+const fs = require('fs');
+const xml2js = require('xml2js');
+const parser = new xml2js.Parser({ attrkey: 'ATTR' });
+const child_process = require('child_process');
+const prompt = require('prompt-sync')();
+
+const specFolder = 'wdio/test';
+
+function GetMetadata(specPath) {
+  const contents = fs.readFileSync(specPath);
+  const metadataTag = '// @metadata ';
+  const metadataStart = contents.indexOf(metadataTag);
+  if (metadataStart != -1) {
+    let metadata = contents
+      .toString()
+      .substr(metadataStart + metadataTag.length)
+      .split(/[\r\n]/)[0];
+    return metadata.split(' ');
+  }
+  return [];
+}
+
+const filters = {
+  SkipCI: specPath => {
+    return process.env.BUILD_QUEUEDBY == 'GitHub';
+  },
+};
+
+// Returns true if the spec is to run.
+// Specs marked SkipCI are excluded from CI (identified by environment variables in the ADO lab)
+function FilterSpec(specPath) {
+  const metadata = GetMetadata(specPath);
+  for (let i = 0; i < metadata.length; i++) {
+    if (filters[metadata[i]](specPath)) {
+      return false;
+    }
+  }
+  return true;
+}
+
+function SelectSpecs(folder) {
+  let specs = [];
+  if (process.argv.length > 2) {
+    specs = process.argv.splice(2).map(spec => spec + '.spec.ts');
+  } else {
+    specs = fs.readdirSync(folder).filter(x => x.endsWith('.spec.ts'));
+  }
+  specs = specs.map(spec => path.join(folder, spec)).filter(FilterSpec);
+  return specs;
+}
+
+let opts = SelectSpecs(specFolder);
+console.log(`Selected tests: ${opts}`);
+
+function OverrideHyperV() {
+  const baseboardMfr = child_process
+    .execSync('powershell.exe (gwmi Win32_BaseBoard).Manufacturer')
+    .toString()
+    .replace(/[\r\n]/, '');
+  if (!baseboardMfr.startsWith('Microsoft Corporation')) {
+    console.log(`Not running in HyperV. Mfr = ${baseboardMfr}`);
+    const answer = prompt(
+      'E2ETest is meant to be run in a HyperV VM. Continue? (Y/N)'
+    );
+    if (answer.toUpperCase() != 'Y') {
+      process.exit(0);
+    }
+  }
+}
+
+OverrideHyperV();
+
+const Launcher = require('@wdio/cli').default;
+
+const wdio = new Launcher('wdio.conf.js', { specs: opts });
+
+function parseLog(logfile) {
+  const xmlString = fs.readFileSync(logfile);
+  let name;
+  parser.parseString(xmlString, (err, res) => {
+    if (!res.testsuites) {
+      name = 'something went wrong';
+    } else {
+      const attr = res.testsuites.testsuite[0].ATTR;
+      if (attr.errors > 0 || attr.failures > 0) {
+        name = attr.name;
+      }
+    }
+  });
+  return name;
+}
+
+function parseLogs() {
+  const reportsDir = path.join(__dirname, 'reports');
+  const logs = fs.readdirSync(reportsDir).filter(x => x.endsWith('.log'));
+  const names = logs
+    .map(x => parseLog(path.join(reportsDir, x)))
+    .filter(x => x != null);
+  return names;
+}
+
+function Process(code) {
+  const failedTests = parseLogs();
+  for (let i = 0; i < failedTests.length; i++) {
+    console.log(`Failed test: ${failedTests[i]}`);
+  }
+  process.exit(code);
+}
+
+wdio.run().then(
+  code => {
+    Process(code);
+  },
+  error => {
+    console.error('Launcher failed to start the test', error.stacktrace);
+    process.exit(1);
+  }
+);