Finish documentation generation feature.

.devcontainer/Dockerfile

@@ -0,0 +1,14 @@
+FROM mcr.microsoft.com/dotnet/core/sdk:3.1-focal
+
+# Mono is required to run pack.ps1, so we install it here.
+RUN apt-get -y update && \
+    apt-get -y install dirmngr gnupg apt-transport-https ca-certificates && \
+    apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF && \
+    sh -c 'echo "deb https://download.mono-project.com/repo/ubuntu stable-focal main" > /etc/apt/sources.list.d/mono-official-stable.list' && \
+    apt-get -y update && \
+    apt-get -y install mono-complete && \
+    apt-get clean && rm -rf /var/lib/apt/lists/
+# We can now get Mono itself.
+RUN curl -o /usr/local/bin/nuget.exe https://dist.nuget.org/win-x86-commandline/latest/nuget.exe && \
+    # Create as alias for nuget
+    echo "alias nuget=\"mono /usr/local/bin/nuget.exe\"" >> /root/.bash_aliases

.devcontainer/README.md

@@ -0,0 +1,21 @@
+# Remote Development Container for Visual Studio Code (preview) #
+
+This folder defines a _development container_ for the Quantum Development Kit to get up and running with your development environment for contributing to the Q# compiler.
+
+## What is a Development Container? ##
+
+Visual Studio Code allows for using [Docker](https://docs.microsoft.com/dotnet/standard/microservices-architecture/container-docker-introduction/docker-defined) to quickly define development environments, including all the compilers, command-line tools, libraries, and programming platforms you need to get up and running quickly.
+Using the definitions provided in this folder, Visual Studio Code can use Docker to automatically install the correct version of the Quantum Development Kit as well as other software you might want to use with Q#, such as Python and Jupyter Notebook --- all into an isolated container that doesn't affect the rest of the software you have on your system.
+
+Next steps:
+- [Visual Studio Code: Developing inside a container](https://code.visualstudio.com/docs/remote/containers)
+
+## Getting Started ##
+
+To use this development container, follow the installation instructions on the [Visual Studio Code site](https://code.visualstudio.com/docs/remote/containers#_installation) to prepare your machine to use development containers such as this one.
+Once you have done so, clone the [**microsoft/quantum**](https://github.com/microsoft/quantum) repository and open the folder in Visual Studio Code.
+You should be prompted to reopen the folder for remote development in the development container; if not, make sure that you have the right extension installed from above.
+
+Once you follow the prompt, Visual Studio Code will automatically configure your development container by installing the Quantum Development Kit into a new image, including installing the .NET Core SDK, project templates, Jupyter Notebook support, and the Python host package.
+This process will take a few moments, but once it's complete, you can then open a new shell as normal in Visual Studio Code; this shell will open a command line in your new development container.
+The Q# compiler will then be available in the `/workspace/qsharp-compiler/` folder of your development container, so you can easily build different parts of the compiler by using `dotnet build`.

.devcontainer/devcontainer.json

@@ -0,0 +1,13 @@
+{
+    "dockerFile": "./Dockerfile",
+    "extensions": [
+        "ms-dotnettools.csharp",
+        "Ionide.Ionide-fsharp",
+        "ms-vscode.powershell",
+        "quantum.quantum-devkit-vscode",
+        "ms-vscode.cpptools"
+    ],
+    "settings": {
+        "terminal.integrated.shell.linux": "/usr/bin/pwsh"
+    }
+}

QsCompiler.sln

@@ -1,3 +1,4 @@
+
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 16
 VisualStudioVersion = 16.0.28809.33
@@ -12,7 +13,7 @@ Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "SyntaxProcessor", "src\QsCo
 EndProject
 Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "TextProcessor", "src\QsCompiler\TextProcessor\TextProcessor.fsproj", "{57F1310F-9210-47A2-AEC5-DE2CF345D754}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DocumentationParser", "src\QsCompiler\DocumentationParser\DocumentationParser.csproj", "{EB06DB1E-2B86-4279-81F9-D39B62B4AE0A}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DocumentationParser", "src\Documentation\DocumentationParser\DocumentationParser.csproj", "{EB06DB1E-2B86-4279-81F9-D39B62B4AE0A}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LanguageServer", "src\QsCompiler\LanguageServer\LanguageServer.csproj", "{D21C9875-2E84-423B-B2CA-84103661F7E4}"
 EndProject
@@ -24,7 +25,7 @@ Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Core", "src\QsCompiler\Core
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Transformations", "src\QsCompiler\Transformations\Transformations.csproj", "{BA5D3733-09F1-4676-9CAC-99AD9A00AECB}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tests.DocGenerator", "src\QsCompiler\Tests.DocGenerator\Tests.DocGenerator.csproj", "{256A6275-FC7F-42E9-9931-BC6EA6D0F31A}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tests.DocGenerator", "src\Documentation\Tests.DocGenerator\Tests.DocGenerator.csproj", "{256A6275-FC7F-42E9-9931-BC6EA6D0F31A}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CommandLineTool", "src\QsCompiler\CommandLineTool\CommandLineTool.csproj", "{B15C4786-0D35-4E66-907F-94DA85FD5576}"
 EndProject
@@ -44,7 +45,9 @@ Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Optimizations", "src\QsComp
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{B4A9484D-31FC-4A27-9E26-4C8DE3E02D77}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Library2", "src\QsCompiler\TestTargets\Libraries\Library2\Library2.csproj", "{E7E019AB-42F8-48AB-B80C-D33351F2C96A}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Library2", "src\QsCompiler\TestTargets\Libraries\Library2\Library2.csproj", "{E7E019AB-42F8-48AB-B80C-D33351F2C96A}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DocumentationGenerator", "src\Documentation\DocumentationGenerator\DocumentationGenerator.csproj", "{11311F0C-78CA-43A3-9C84-D7EA0AE9748C}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -272,6 +275,18 @@ Global
 		{E7E019AB-42F8-48AB-B80C-D33351F2C96A}.Release|x64.Build.0 = Release|Any CPU
 		{E7E019AB-42F8-48AB-B80C-D33351F2C96A}.Release|x86.ActiveCfg = Release|Any CPU
 		{E7E019AB-42F8-48AB-B80C-D33351F2C96A}.Release|x86.Build.0 = Release|Any CPU
+		{11311F0C-78CA-43A3-9C84-D7EA0AE9748C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{11311F0C-78CA-43A3-9C84-D7EA0AE9748C}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{11311F0C-78CA-43A3-9C84-D7EA0AE9748C}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{11311F0C-78CA-43A3-9C84-D7EA0AE9748C}.Debug|x64.Build.0 = Debug|Any CPU
+		{11311F0C-78CA-43A3-9C84-D7EA0AE9748C}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{11311F0C-78CA-43A3-9C84-D7EA0AE9748C}.Debug|x86.Build.0 = Debug|Any CPU
+		{11311F0C-78CA-43A3-9C84-D7EA0AE9748C}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{11311F0C-78CA-43A3-9C84-D7EA0AE9748C}.Release|Any CPU.Build.0 = Release|Any CPU
+		{11311F0C-78CA-43A3-9C84-D7EA0AE9748C}.Release|x64.ActiveCfg = Release|Any CPU
+		{11311F0C-78CA-43A3-9C84-D7EA0AE9748C}.Release|x64.Build.0 = Release|Any CPU
+		{11311F0C-78CA-43A3-9C84-D7EA0AE9748C}.Release|x86.ActiveCfg = Release|Any CPU
+		{11311F0C-78CA-43A3-9C84-D7EA0AE9748C}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

bootstrap.ps1

@@ -1,3 +1,4 @@
+[CmdletBinding()]
 param(
     [string]
     $AssemblyVersion = $Env:ASSEMBLY_VERSION,
@@ -68,6 +69,7 @@ If ($Env:SEMVER_VERSION -eq $null) { $Env:SEMVER_VERSION ="$SemverVersion" }
 If ($Env:VSVSIX_VERSION -eq $null) { $Env:VSVSIX_VERSION ="$VsVsixVersion" }
 Write-Host "##vso[task.setvariable variable=VsVsix.Version]$VsVsixVersion"
 
+Write-Host "##[info]Finding NuSpec references..."
 Push-Location (Join-Path $PSScriptRoot 'src/QsCompiler/Compiler')
 .\FindNuspecReferences.ps1;
 Pop-Location

build/build.ps1

@@ -121,6 +121,11 @@ if ($Env:ENABLE_VSIX -ne "false") {
     Write-Host "##vso[task.logissue type=warning;]VSIX building skipped due to ENABLE_VSIX variable."
 }
 
+# NB: In other repos, we check the manifest here. That can cause problems
+#     in our case, however, as some assemblies are only produced during
+#     packing and publishing. Thus, as an exception, we verify the manifest
+#     only in pack.ps1.
+
 if (-not $all_ok) {
     throw "Building failed. Check the logs."
     exit 1

build/manifest.ps1

@@ -0,0 +1,48 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+#!/usr/bin/env pwsh
+#Requires -PSEdition Core
+
+& "$PSScriptRoot/set-env.ps1"
+
+if ($Env:ENABLE_VSIX -ne "false") {
+    # The language server is only built if either the VS2019 or VS Code extension
+    # is enabled.
+    $VsixAssemblies = @(
+        "./src/QsCompiler/LanguageServer/bin/$Env:BUILD_CONFIGURATION/netcoreapp3.1/Microsoft.Quantum.QsCompilationManager.dll",
+        "./src/QsCompiler/LanguageServer/bin/$Env:BUILD_CONFIGURATION/netcoreapp3.1/Microsoft.Quantum.QsCore.dll",
+        "./src/QsCompiler/LanguageServer/bin/$Env:BUILD_CONFIGURATION/netcoreapp3.1/Microsoft.Quantum.QsDataStructures.dll",
+        "./src/QsCompiler/LanguageServer/bin/$Env:BUILD_CONFIGURATION/netcoreapp3.1/Microsoft.Quantum.QsLanguageServer.dll",
+        "./src/QsCompiler/LanguageServer/bin/$Env:BUILD_CONFIGURATION/netcoreapp3.1/Microsoft.Quantum.QsSyntaxProcessor.dll",
+        "./src/QsCompiler/LanguageServer/bin/$Env:BUILD_CONFIGURATION/netcoreapp3.1/Microsoft.Quantum.QsTextProcessor.dll",
+        "./src/QsCompiler/LanguageServer/bin/$Env:BUILD_CONFIGURATION/netcoreapp3.1/Microsoft.Quantum.QsTransformations.dll"
+    );
+
+    # The VS2019 extension itself is only built if msbuild is present.
+    if (Get-Command msbuild -ErrorAction SilentlyContinue) {
+        $VsixAssemblies += @(
+            "./src/VisualStudioExtension/QsharpVSIX/bin/$Env:BUILD_CONFIGURATION/Microsoft.Quantum.VisualStudio.Extension.dll"
+        );
+    }
+} else {
+    $VsixAssemblies = @();
+}
+
+
+@{
+    Packages = @(
+        "Microsoft.Quantum.Compiler",
+        "Microsoft.Quantum.Compiler.CommandLine",
+        "Microsoft.Quantum.ProjectTemplates",
+        "Microsoft.Quantum.Sdk",
+        "Microsoft.Quantum.DocumentationGenerator"
+    );
+    Assemblies = $VsixAssemblies + @(
+        "./src/Documentation/DocumentationGenerator/bin/$Env:BUILD_CONFIGURATION/netstandard2.1/Microsoft.Quantum.DocumentationGenerator.dll",
+        "./src/QsCompiler/CommandLineTool/bin/$Env:BUILD_CONFIGURATION/netcoreapp3.1/Microsoft.Quantum.QsCompiler.dll",
+        "./src/QsCompiler/CommandLineTool/bin/$Env:BUILD_CONFIGURATION/netcoreapp3.1/Microsoft.Quantum.QsDocumentationParser.dll",
+        "./src/QsCompiler/CommandLineTool/bin/$Env:BUILD_CONFIGURATION/netcoreapp3.1/qsc.dll",
+        "./src/QuantumSdk/Tools/BuildConfiguration/bin/$Env:BUILD_CONFIGURATION/netcoreapp3.1/publish/Microsoft.Quantum.Sdk.BuildConfiguration.dll"
+    ) | ForEach-Object { Get-Item (Join-Path $PSScriptRoot ".." $_) };
+} | Write-Output;

build/pack.ps1

@@ -54,6 +54,30 @@ function Pack-One() {
     }
 }
 
+function Pack-Dotnet() {
+    Param($project, $option1 = "", $option2 = "", $option3 = "")
+    if ("" -ne "$Env:ASSEMBLY_CONSTANTS") {
+        $args = @("/property:DefineConstants=$Env:ASSEMBLY_CONSTANTS");
+    }  else {
+        $args = @();
+    }
+    dotnet pack (Join-Path $PSScriptRoot $project) `
+        -o $Env:NUGET_OUTDIR `
+        -c $Env:BUILD_CONFIGURATION `
+        -v detailed `
+        @args `
+        /property:Version=$Env:ASSEMBLY_VERSION `
+        /property:PackageVersion=$Env:NUGET_VERSION `
+        $option1 `
+        $option2 `
+        $option3
+
+    if ($LastExitCode -ne 0) {
+        Write-Host "##vso[task.logissue type=error;]Failed to pack $project."
+        $script:all_ok = $False
+    }
+}
+
 ##
 # Q# Language Server (self-contained)
 ##
@@ -213,6 +237,7 @@ Publish-One '../src/QuantumSdk/Tools/BuildConfiguration/BuildConfiguration.cspro
 
 Pack-One '../src/QsCompiler/Compiler/Compiler.csproj' '-IncludeReferencedProjects'
 Pack-One '../src/QsCompiler/CommandLineTool/CommandLineTool.csproj' '-IncludeReferencedProjects'
+Pack-Dotnet '../src/Documentation/DocumentationGenerator/DocumentationGenerator.csproj'
 Pack-One '../src/ProjectTemplates/Microsoft.Quantum.ProjectTemplates.nuspec'
 Pack-One '../src/QuantumSdk/QuantumSdk.nuspec'
 
@@ -230,6 +255,18 @@ if ($Env:ENABLE_VSIX -ne "false") {
     Write-Host "##vso[task.logissue type=warning;]VSIX packing skipped due to ENABLE_VSIX variable."
 }
 
+# Copy documentation summarization tool into docs drop.
+# Note that we only copy this tool when DOCS_OUTDIR is set (that is, when we're
+# collecting docs in a build artifact).
+if ("$Env:DOCS_OUTDIR".Trim() -ne "") {
+    Push-Location (Join-Path $PSScriptRoot "../src/Documentation/Summarizer")
+        Copy-Item -Path *.py, *.txt -Destination $Env:DOCS_OUTDIR
+    Pop-Location
+}
+
+Write-Host "##[info]Verifying manifest..."
+& (Join-Path $PSScriptRoot "manifest.ps1")
+
 if (-not $all_ok) {
     throw "Packing failed. Check the logs."
     exit 1

src/Documentation/DocumentationGenerator/DocumentationGeneration.cs

@@ -0,0 +1,93 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
+using System.Collections.Generic;
+using Microsoft.CodeAnalysis;
+using Microsoft.Quantum.QsCompiler;
+using Microsoft.Quantum.QsCompiler.SyntaxTree;
+
+namespace Microsoft.Quantum.Documentation
+{
+    /// <summary>
+    ///     Rewrite step that generates API documentation from documentation
+    ///     comments in the Q# source files being compiled.
+    /// </summary>
+    public class DocumentationGeneration : IRewriteStep
+    {
+        private readonly List<IRewriteStep.Diagnostic> diagnostics;
+
+        /// <summary>
+        ///     Initializes a new instance of the <see cref="DocumentationGeneration"/> class.
+        /// </summary>
+        public DocumentationGeneration()
+        {
+            this.diagnostics = new List<IRewriteStep.Diagnostic>(); // collects diagnostics that will be displayed to the user
+        }
+
+        /// <inheritdoc/>
+        public string Name => "DocumentationGeneration";
+
+        /// <inheritdoc/>
+        public int Priority => 0; // only compared within this dll
+
+        /// <inheritdoc/>
+        public IDictionary<string, string?> AssemblyConstants { get; } = new Dictionary<string, string?>();
+
+        /// <inheritdoc/>
+        public IEnumerable<IRewriteStep.Diagnostic> GeneratedDiagnostics => this.diagnostics;
+
+        /// <inheritdoc/>
+        public bool ImplementsPreconditionVerification => true;
+
+        /// <inheritdoc/>
+        public bool ImplementsTransformation => true;
+
+        /// <inheritdoc/>
+        public bool ImplementsPostconditionVerification => false;
+
+        /// <inheritdoc/>
+        public bool PreconditionVerification(QsCompilation compilation)
+        {
+            var preconditionPassed = true; // nothing to check
+            if (preconditionPassed)
+            {
+                // Diagnostics with severity Info or lower usually won't be displayed to the user.
+                // If the severity is Error or Warning the diagnostic is shown to the user like any other compiler diagnostic,
+                // and if the Source property is set to the absolute path of an existing file,
+                // the user will be directed to the file when double clicking the diagnostics.
+                this.diagnostics.Add(new IRewriteStep.Diagnostic
+                {
+                    Severity = DiagnosticSeverity.Info,
+                    Message = $"Precondition for {this.Name} was {(preconditionPassed ? "satisfied" : "not satisfied")}.",
+                    Stage = IRewriteStep.Stage.PreconditionVerification,
+                });
+            }
+
+            return preconditionPassed;
+        }
+
+        public bool Transformation(QsCompilation compilation, out QsCompilation transformed)
+        {
+            var docProcessor = new ProcessDocComments(
+                this.AssemblyConstants.TryGetValue("DocsOutputPath", out var path)
+                ? path
+                : null,
+                this.AssemblyConstants.TryGetValue("DocsPackageId", out var packageName)
+                ? packageName
+                : null
+            );
+
+            docProcessor.OnDiagnostic += diagnostic =>
+            {
+                this.diagnostics.Add(diagnostic);
+            };
+
+            transformed = docProcessor.OnCompilation(compilation);
+            return true;
+        }
+
+        public bool PostconditionVerification(QsCompilation compilation) =>
+            throw new NotImplementedException();
+    }
+}

src/Documentation/DocumentationGenerator/DocumentationGenerator.csproj

@@ -0,0 +1,43 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <Import Project="..\..\Common\AssemblyCommon.props" />
+
+  <PropertyGroup>
+    <TargetFramework>netstandard2.1</TargetFramework>
+    <AssemblyName>Microsoft.Quantum.DocumentationGenerator</AssemblyName>
+    <Nullable>enable</Nullable>
+    <WarningsAsErrors>nullable</WarningsAsErrors>
+  </PropertyGroup>  
+
+  <PropertyGroup>
+    <Authors>Microsoft</Authors>
+    <Description>Experimental documentation generation tool for Q#.</Description>
+    <PackageName>Microsoft.Quantum.DocumentationGenerator</PackageName>
+    <PackageReleaseNotes>See: https://docs.microsoft.com/en-us/quantum/relnotes/</PackageReleaseNotes>
+    <PackageLicenseExpression>MIT</PackageLicenseExpression>
+    <PackageProjectUrl>https://github.com/microsoft/qsharp-compiler</PackageProjectUrl>
+    <PackageIconUrl>https://secure.gravatar.com/avatar/bd1f02955b2853ba0a3b1cdc2434e8ec.png</PackageIconUrl>
+    <PackageTags>Quantum Q# Qsharp</PackageTags>
+    <PublishRepositoryUrl>true</PublishRepositoryUrl>
+  </PropertyGroup>  
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\QsCompiler\Compiler\Compiler.csproj" />
+    <PackageReference Include="StyleCop.Analyzers" Version="1.2.0-beta.164" PrivateAssets="All" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Compile Include="..\..\Common\DelaySign.cs" Link="Properties\DelaySign.cs" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Content Include="DocumentationGenerator.props">
+      <Pack>true</Pack>
+      <PackagePath>build\Microsoft.Quantum.DocumentationGenerator.props</PackagePath>
+    </Content>
+  </ItemGroup>
+
+  <ItemGroup>
+    <AdditionalFiles Include="..\..\Common\stylecop.json" Link="stylecop.json" />
+  </ItemGroup>
+
+</Project>

src/Documentation/DocumentationGenerator/DocumentationGenerator.props

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+
+  <PropertyGroup>
+    <QscRef_Microsoft_Quantum_DocumentationGenerator>
+      $(MSBuildThisFileDirectory)/../lib/netstandard2.1/Microsoft.Quantum.DocumentationGenerator.dll
+    </QscRef_Microsoft_Quantum_DocumentationGenerator>
+  </PropertyGroup>
+
+</Project>