Split out Smoke Tests and Unit Tests into a separate Jobs

SmokeTests/SmokeTest.csproj

@@ -112,18 +112,18 @@
       <Version>2.4.3</Version>
     </PackageReference>
   </ItemGroup>
-  <ItemGroup Condition="'$(CurrentProject)' != ''">
+  <ItemGroup Condition="'$(CurrentProject)' != '' and '$(CurrentProject)' != 'UWPBaseline'">
     <PackageReference Include="$(CurrentProject)" Version="7.*-*" />
   </ItemGroup>
   <PropertyGroup Condition=" '$(VisualStudioVersion)' == '' or '$(VisualStudioVersion)' &lt; '14.0' ">
     <VisualStudioVersion>14.0</VisualStudioVersion>
   </PropertyGroup>
   <Import Project="$(MSBuildExtensionsPath)\Microsoft\WindowsXaml\v$(VisualStudioVersion)\Microsoft.Windows.UI.Xaml.CSharp.targets" />
   <Target Name="BeforeBuild">
-    <ItemGroup>
+    <ItemGroup Condition="$(CurrentProject) != 'UWPBaseline'">
       <ToolkitNugets Include="../bin/nupkg/$(CurrentProject).*.nupkg"/>
       <ToolkitNuget Condition="$([System.Text.RegularExpressions.Regex]::IsMatch('%(Identity)', `$(CurrentProject).([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?.nupkg`))" Include="@(ToolkitNugets)"/>
     </ItemGroup>
-    <Error Condition="'@(ToolkitNuget)' == ''" Text="NuGet $(CurrentProject).[SEMVER].nupkg doesn't exist!"/>
+    <Error Condition="'@(ToolkitNuget)' == '' and $(CurrentProject) != 'UWPBaseline'" Text="NuGet $(CurrentProject).[SEMVER].nupkg doesn't exist!"/>
   </Target>
 </Project>

SmokeTests/SmokeTestAnalysis.ps1

@@ -0,0 +1,149 @@
+$ErrorActionPreference = "Stop"
+$ProgressPreference="SilentlyContinue"
+
+Write-Host "Running Smoke Test Package Analyis"
+Write-Host "----------------------------------"
+
+# Our script is at our SmokeTest root, we want to look for the AppPackages folder
+$PackagePath = $PSScriptRoot + "\AppPackages\"
+$FilePattern = "SmokeTest_{0}_x86_bundle.msixupload"
+$BaselineName = $FilePattern -f "UWPBaseline"
+$TempFolder = "ExplodedArchive"
+
+function Expand-MsixUploadPackage {
+    param (
+        [string]$PackageFile,
+        [string]$Destination
+    )
+
+    $ZipUpload = $PackageFile.Replace("msixupload", "zip")
+
+    Move-Item $PackageFile -Destination $ZipUpload
+
+    Expand-Archive $ZipUpload -DestinationPath $Destination
+
+    Move-Item $ZipUpload -Destination $PackageFile
+
+    Push-Location $Destination
+
+    $Bundle = (Get-ChildItem "*.msixbundle").Name
+    $ZipBundle = $Bundle.Replace("msixbundle", "zip")
+
+    Move-Item $Bundle -Destination $ZipBundle
+
+    Expand-Archive $ZipBundle -DestinationPath .
+
+    Remove-Item $ZipBundle
+
+    $msix = (Get-ChildItem "*.msix").Name
+    $ZipMSIX = $msix.Replace("msix", "zip")
+
+    Move-Item $msix -Destination $ZipMSIX
+
+    Expand-Archive $ZipMSIX -DestinationPath . -Force # Force here as we have some duplicate file names we don't really care about from parent archives
+
+    Remove-Item $ZipMSIX
+
+    Pop-Location
+}
+
+if (Test-Path $PackagePath)
+{    
+    Push-Location $PackagePath
+
+    Write-Host "Extracting Baseline..."
+
+    # TODO: Theoretically we could grab bits from the bin directory instead of having to expand each package, not sure about what we ignore though
+    Expand-MsixUploadPackage $BaselineName -Destination $TempFolder
+
+    # Get all the base file info only (grab stuff in directories but not the directories themselves)
+    $BaselineFiles = Get-ChildItem $TempFolder -Recurse -Attributes !Directory -Exclude "SmokeTest*"
+    $SmokeTestFiles = Get-ChildItem $TempFolder -Recurse -Attributes !Directory -Include "SmokeTest*"
+
+    $BaselineFootprint = ($BaselineFiles | Measure-Object -Property Length -sum).Sum + ($SmokeTestFiles | Measure-Object -Property Length -sum).Sum
+    Write-Host ("Baseline Footprint: {0:n0} bytes" -f $BaselineFootprint)
+    Write-Host "-----------------------------------------"
+
+    $PackageList = Get-ChildItem "$PackagePath*.msixupload" -Exclude $BaselineName
+
+    #$i = 0
+    foreach ($Package in $PackageList)
+    {
+        #Write-Progress -Id 0 -Activity "Comparing Against Baseline..." -Status "Prepping Package" -PercentComplete (($i++ / $PackageList.count)*100) -CurrentOperation $Package.Name
+
+        # Make sure we've cleaned-up the last archive
+        Remove-Item $TempFolder -Recurse -Force
+
+        #$ProgressPreference="SilentlyContinue"
+        Expand-MsixUploadPackage $Package.Name -Destination $TempFolder
+        #$ProgressPreference="Continue"
+
+        [System.Collections.ArrayList]$PackageFiles = Get-ChildItem $TempFolder -Recurse -Attributes !Directory -Exclude "SmokeTest*"
+        $PackageSmokeTestFiles = Get-ChildItem $TempFolder -Recurse -Attributes !Directory -Include "SmokeTest*"
+
+        # TODO: Make function or regex better to extra package name more easily based on a template string at the top or something...
+        $PackageShortName = $Package.Name.substring(10, $Package.Name.Length - 32)
+        Write-Host ("{0} Additional Footprint: {1:n0} bytes" -f $PackageShortName, (($PackageFiles | Measure-Object -Property Length -sum).Sum + ($PackageSmokeTestFiles | Measure-Object -Property Length -sum).Sum - $BaselineFootprint))
+
+        # Quick check on the base exe file/symbols differences
+        foreach ($file in $SmokeTestFiles)
+        {
+            $match = $null
+            $match = $PackageSmokeTestFiles | Where-Object {$_.Extension -eq $file.Extension}
+            if ($null -ne $match)
+            {
+                Write-Host ("  App Diff: ({0}) = {1:n0}" -f $file.Extension, ($match.Length - $file.Length)) -ForegroundColor DarkCyan
+            }
+        }
+
+        #$j = 0
+        foreach ($file in $BaselineFiles)
+        {
+            #Write-Progress -Id 1 -ParentId 0 -Activity "Comparing Against Baseline..." -Status "Comparing Package" -PercentComplete (($j++ / $BaselineFiles.count)*100) -CurrentOperation $file.Name
+
+            $match = $null
+            $match = $PackageFiles | Where-Object {$_.Name -eq $file.Name}
+            if ($null -ne $match)
+            {
+                # File was in baseline, but has a different size
+                if ($match.Length -ne $file.Length)
+                {
+                    Write-Host ("  Size Diff: {0} = {1:n0}" -f $file.Name, ($match.Length - $file.Length)) -ForegroundColor Magenta
+                }
+
+                # Remove checked files (performance) and also remaining are new
+                $PackageFiles.Remove($match)
+            }
+        }
+
+        # List remaining (new) files to this package
+        foreach ($file in $PackageFiles)
+        {
+            if ($file.Name -match $PackageShortName)
+            {
+                Write-Host ("  Lib (self): {0} = {1:n0}" -f $file.Name, $file.Length) -ForegroundColor White
+            }
+            else 
+            {
+                Write-Host ("  Additional: {0} = {1:n0}" -f $file.Name, $file.Length) -ForegroundColor Yellow
+            }
+        }
+
+        # TODO: Especially if we add comparison to the main branch, we should format as an actual table and colorize via VT: https://stackoverflow.com/a/49038815/8798708
+
+        #Write-Progress -Id 1 -ParentId 0 -Activity "Comparing Against Baseline..." -Completed
+        Write-Host "-----------------------------------------"
+        Write-Host
+    }
+
+    #Write-Progress -Id 0 -Activity "Comparing Against Baseline..." -Completed
+
+    # Clean-up
+    Remove-Item $TempFolder -Recurse -Force
+
+    Pop-Location
+}
+else 
+{
+    Write-Error "Path $PackagePath not found for analysis!"        
+}

SmokeTests/SmokeTests.proj

@@ -4,15 +4,15 @@
   <PropertyGroup>
     <BuildPlatforms>x86</BuildPlatforms>
     <BuildConfigurations>Release</BuildConfigurations>
-    <ToolkitPackages>Microsoft.Toolkit;Microsoft.Toolkit.HighPerformance;Microsoft.Toolkit.Parsers;Microsoft.Toolkit.Mvvm;Microsoft.Toolkit.Services;Microsoft.Toolkit.Uwp;Microsoft.Toolkit.Uwp.Connectivity;Microsoft.Toolkit.Uwp.DeveloperTools;Microsoft.Toolkit.Uwp.Input.GazeInteraction;Microsoft.Toolkit.Uwp.Notifications;Microsoft.Toolkit.Uwp.UI;Microsoft.Toolkit.Uwp.UI.Animations;Microsoft.Toolkit.Uwp.UI.Controls;Microsoft.Toolkit.Uwp.UI.Controls.DataGrid;Microsoft.Toolkit.Uwp.UI.Controls.Layout;Microsoft.Toolkit.Uwp.UI.Media;Microsoft.Toolkit.Uwp.UI.Controls.Markdown</ToolkitPackages>
+    <ToolkitPackages>UWPBaseline;Microsoft.Toolkit;Microsoft.Toolkit.HighPerformance;Microsoft.Toolkit.Parsers;Microsoft.Toolkit.Mvvm;Microsoft.Toolkit.Services;Microsoft.Toolkit.Uwp;Microsoft.Toolkit.Uwp.Connectivity;Microsoft.Toolkit.Uwp.DeveloperTools;Microsoft.Toolkit.Uwp.Input.GazeInteraction;Microsoft.Toolkit.Uwp.Notifications;Microsoft.Toolkit.Uwp.UI;Microsoft.Toolkit.Uwp.UI.Animations;Microsoft.Toolkit.Uwp.UI.Controls;Microsoft.Toolkit.Uwp.UI.Controls.DataGrid;Microsoft.Toolkit.Uwp.UI.Controls.Layout;Microsoft.Toolkit.Uwp.UI.Media;Microsoft.Toolkit.Uwp.UI.Controls.Markdown</ToolkitPackages>
   </PropertyGroup>
 
   <Target Name="Build" 
           DependsOnTargets="ChooseProjectsToBuild"
           Inputs="@(ProjectsToBuild)"
           Outputs="%(Filename)">
 
-    <Message Importance="High" Text="Building project %(ProjectsToBuild.Filename): (%(ProjectsToBuild.Configuration)|%(ProjectsToBuild.Platform))" />
+    <Message Importance="High" Text="Building project %(ProjectsToBuild.Identity): (%(ProjectsToBuild.Configuration)|%(ProjectsToBuild.Platform))" />
 
     <MSBuild Projects="SmokeTest.csproj"
               Targets="restore;build"

SmokeTests/UWPBaseline/MainPage.xaml

@@ -0,0 +1,12 @@
+<Page
+    x:Class="SmokeTest.MainPage"
+    xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
+    xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
+    xmlns:local="using:SmokeTest"
+    xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
+    xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
+    mc:Ignorable="d"
+    Background="{ThemeResource ApplicationPageBackgroundThemeBrush}">
+
+    <Grid/>
+</Page>

SmokeTests/UWPBaseline/MainPage.xaml.cs

@@ -0,0 +1,14 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+namespace SmokeTest
+{
+    public sealed partial class MainPage
+    {
+        public MainPage()
+        {
+            InitializeComponent();
+        }
+    }
+}

azure-pipelines.yml

@@ -13,8 +13,9 @@ variables:
   BuildConfiguration: Release
 
 jobs:
-- job: ToolkitBuild
-  timeoutInMinutes: 120
+### BUILD ###
+- job: BuildBits
+  timeoutInMinutes: 60
 
   steps:
   - task: BatchScript@1
@@ -45,15 +46,10 @@ jobs:
   - powershell: .\build\build.ps1 -target=Build
     displayName: Build
 
+  ### Unit Tests ###
+
   - powershell: .\build\build.ps1 -target=Test
     displayName: Test
-    timeoutInMinutes: 15
-
-  - powershell: .\build\build.ps1 -target=Package
-    displayName: Package
-
-  - powershell: .\build\build.ps1 -target=SmokeTest
-    displayName: SmokeTest
 
   - task: PublishTestResults@2
     inputs:
@@ -62,19 +58,29 @@ jobs:
     displayName: Publish Test Results
     condition: always()
 
+  - task: PublishPipelineArtifact@1
+    displayName: Publish Test WexLogFileOutput
+    inputs:
+      targetPath: .\build\WexLogFileOutput
+      artifactName: WexUnitTestErrorLogFileOutput
+    condition: failed()
+
+  ### UI Integration Tests ###
+
+  - powershell: .\build\build.ps1 -target=UITest
+    displayName: UI Integration Tests
+
   - task: PublishPipelineArtifact@1
     displayName: Publish UI Test Results
     inputs:
       targetPath: .\build\UITestResults.wtl
-      artifactName: WexLogFileOutput
+      artifactName: WexUITestLogFileOutput
     condition: always()
 
-  - task: PublishPipelineArtifact@1
-    displayName: Publish Test WexLogFileOutput
-    inputs:
-      targetPath: .\build\WexLogFileOutput
-      artifactName: WexErrorLogFileOutput
-    condition: failed()
+  ### Package ###
+
+  - powershell: .\build\build.ps1 -target=Package
+    displayName: Package
 
   - task: PowerShell@2
     displayName: Authenticode Sign Packages
@@ -86,13 +92,28 @@ jobs:
       ArtifactDirectory: bin\nupkg
     condition: and(succeeded(), not(eq(variables['build.reason'], 'PullRequest')), not(eq(variables['SignClientSecret'], '')), not(eq(variables['SignClientUser'], '')))
 
-  - task: PublishBuildArtifacts@1
+  - task: PublishPipelineArtifact@1
     displayName: Publish Package Artifacts
     inputs:
-      pathToPublish: .\bin\nupkg
-      artifactType: container
+      targetPath: .\bin\nupkg
       artifactName: Packages
 
+### Smoke Tests ###
+
+- job: SmokeTests
+  dependsOn: BuildBits
+  timeoutInMinutes: 40
+
+  steps:
+  - task: DownloadPipelineArtifact@2
+    displayName: Download NuGet Packages Artifact
+    inputs:
+      artifact: Packages
+      path: .\bin\nupkg
+
+  - powershell: .\build\build.ps1 -target=SmokeTest
+    displayName: SmokeTest
+
   - task: CopyFiles@2
     inputs:
       sourceFolder: .\SmokeTests\AppPackages
@@ -105,3 +126,6 @@ jobs:
       pathToPublish: $(build.artifactstagingdirectory)\SmokeTestBundles
       artifactType: container
       artifactName: SmokeTestBundles
+
+  - powershell: .\SmokeTests\SmokeTestAnalysis.ps1
+    displayName: Analyze Package Sizes

build/build.cake

@@ -243,7 +243,7 @@ public string getMSTestAdapterPath(){
 }
 
 Task("Test")
-	.Description("Runs all Tests")
+	.Description("Runs all Unit Tests")
     .Does(() =>
 {
     Information("\nRunning Unit Tests");
@@ -272,7 +272,11 @@ Task("Test")
         ArgumentCustomization = arg => arg.Append($"-s {baseDir}/.runsettings"),
     };
     DotNetCoreTest(file.FullPath, testSettings);
-}).DoesForEach(GetFiles(taefBinDir + "/**/UITests.Tests.TAEF.dll"), (file) =>
+}).DeferOnError();
+
+Task("UITest")
+	.Description("Runs all UI Tests")
+    .DoesForEach(GetFiles(taefBinDir + "/**/UITests.Tests.TAEF.dll"), (file) =>
 {
     Information("\nRunning TAEF Interaction Tests");
 
@@ -315,6 +319,7 @@ Task("MSTestUITest")
 Task("Default")
     .IsDependentOn("Build")
     .IsDependentOn("Test")
+    .IsDependentOn("UITest")
     .IsDependentOn("Package");
 
 Task("UpdateHeaders")