From f2d69c202298a23d594a03840337aed58811ee5d Mon Sep 17 00:00:00 2001 From: Christopher Granade Date: Tue, 7 Apr 2020 13:41:18 -0700 Subject: [PATCH 01/17] First stab at automatically documenting magic commands. --- build/docs/README.md | 4 + build/docs/build_docs.py | 123 ++++++++++++++++++ src/Jupyter/Extensions.cs | 1 + src/Jupyter/IQSharpEngine.cs | 5 +- src/Jupyter/Magic/LsMagicMagic.cs | 48 +++++++ .../Magic/Resolution/IMagicResolver.cs | 29 +++++ .../{ => Magic/Resolution}/MagicResolver.cs | 14 +- src/Python/qsharp/packages.py | 4 +- 8 files changed, 219 insertions(+), 9 deletions(-) create mode 100644 build/docs/README.md create mode 100644 build/docs/build_docs.py create mode 100644 src/Jupyter/Magic/LsMagicMagic.cs create mode 100644 src/Jupyter/Magic/Resolution/IMagicResolver.cs rename src/Jupyter/{ => Magic/Resolution}/MagicResolver.cs (91%) diff --git a/build/docs/README.md b/build/docs/README.md new file mode 100644 index 0000000000..5819103327 --- /dev/null +++ b/build/docs/README.md @@ -0,0 +1,4 @@ +# Building IQ# Reference Documentation + +The contents of this folder automatically builds documentation for each +available magic command. diff --git a/build/docs/build_docs.py b/build/docs/build_docs.py new file mode 100644 index 0000000000..977760437f --- /dev/null +++ b/build/docs/build_docs.py @@ -0,0 +1,123 @@ +import json +import datetime + +from io import StringIO +from typing import List, Optional +from pathlib import Path + +import click + +try: + import ruamel.yaml as yaml +except ImportError: + import ruamel_yaml as yaml + +@click.command() +@click.argument("OUTPUT_DIR") +@click.argument("UID_BASE") +@click.option("--package", "-p", multiple=True) +def main(output_dir : str, uid_base : str, package : List[str]): + output_dir = Path(output_dir) + # Make the output directory if it doesn't already exist. + output_dir.mkdir(parents=True, exist_ok=True) + + import qsharp + + print("Adding packages...") + for package_name in package: + qsharp.packages.add(package_name) + + print("Generating Markdown files...") + magics = qsharp.client._execute(r"%lsmagic") + for magic in magics: + magic_doc = format_as_document(magic, uid_base) + with open(output_dir / f"{magic['Name'].replace('%', '')}.md", 'w', encoding='utf8') as f: + f.write(magic_doc) + +def format_as_section(name : str, content : Optional[str]) -> str: + content = content.strip() if content else None + return f""" + +## {name} + +{content} + +""" if content else "" + +def _cleanup_markdown(content : str): + # Ensure that there is exactly one trailing newline, and that each line + # is free of trailing whitespace (with an exception for exactly two + # trailing spaces). + # We also want to make sure that there are not two or more blank lines in + # a row. + prev_blank = False + for line in content.split("\n"): + cleaned_line = line.rstrip() if line != line.rstrip() + " " else line + this_blank = cleaned_line == "" + + if not (prev_blank and this_blank): + yield cleaned_line + + prev_blank = this_blank + +def cleanup_markdown(content : str): + return "\n".join( + _cleanup_markdown(content) + ).strip() + "\n" + +def format_as_document(magic, uid_base : str) -> str: + # NB: this function supports both the old and new Documentation format. + # See https://github.com/microsoft/jupyter-core/pull/49. + magic_name = magic['Name'].strip() + metadata = { + 'title': f"{magic_name} (magic command)", + 'uid': f"{uid_base}.{magic_name.replace('%', '')}", + 'ms.date': datetime.date.today().isoformat(), + 'ms.topic': 'article' + } + header = f"# `{magic_name}`" + doc = magic['Documentation'] + + summary = format_as_section('Summary', doc.get('Summary', "")) + description = format_as_section( + 'Description', + doc.get('Description', doc.get('Full', '')) + ) + remarks = format_as_section('Remarks', doc.get('Remarks', "")) + examples = "\n".join( + format_as_section("Example", example) + for example in doc.get('Examples', []) + ) + see_also = format_as_section( + "See Also", + "\n".join( + f"- [{description}]({target})" + for description, target in doc.get('SeeAlso', []) + ) + ) + + # Convert the metadata header to YAML. + metadata_as_yaml = StringIO() + yaml.dump(metadata, metadata_as_yaml) + + return cleanup_markdown(f""" +--- +{metadata_as_yaml.getvalue()} +--- + +{header} +{summary} +{description} +{remarks} +{examples} +{see_also} + """) + +if __name__ == "__main__": + main() diff --git a/src/Jupyter/Extensions.cs b/src/Jupyter/Extensions.cs index 624b4935f6..f202ec4383 100644 --- a/src/Jupyter/Extensions.cs +++ b/src/Jupyter/Extensions.cs @@ -35,6 +35,7 @@ public static ChannelWithNewLines WithNewLines(this IChannel original) => public static void AddIQSharpKernel(this IServiceCollection services) { services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); } diff --git a/src/Jupyter/IQSharpEngine.cs b/src/Jupyter/IQSharpEngine.cs index 758a9bb1de..19a696fbd6 100644 --- a/src/Jupyter/IQSharpEngine.cs +++ b/src/Jupyter/IQSharpEngine.cs @@ -38,7 +38,8 @@ public IQSharpEngine( IConfigurationSource configurationSource, PerformanceMonitor performanceMonitor, IShellRouter shellRouter, - IEventService eventService + IEventService eventService, + IMagicSymbolResolver magicSymbolResolver ) : base(shell, context, logger) { this.performanceMonitor = performanceMonitor; @@ -46,7 +47,7 @@ IEventService eventService this.Snippets = services.GetService(); this.SymbolsResolver = services.GetService(); - this.MagicResolver = new MagicSymbolResolver(services, logger); + this.MagicResolver = magicSymbolResolver; RegisterDisplayEncoder(new IQSharpSymbolToHtmlResultEncoder()); RegisterDisplayEncoder(new IQSharpSymbolToTextResultEncoder()); diff --git a/src/Jupyter/Magic/LsMagicMagic.cs b/src/Jupyter/Magic/LsMagicMagic.cs new file mode 100644 index 0000000000..1306bb6a08 --- /dev/null +++ b/src/Jupyter/Magic/LsMagicMagic.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Linq; + +using Microsoft.Jupyter.Core; +using Microsoft.Quantum.IQSharp; +using Microsoft.Quantum.IQSharp.Jupyter; + +namespace Microsoft.Quantum.IQSharp.Jupyter +{ + /// + /// A magic command that lists what magic commands are currently + /// available. + /// + public class LsMagicMagic : AbstractMagic + { + private readonly IMagicSymbolResolver resolver; + /// + /// Given a given snippets collection, constructs a new magic command + /// that queries callables defined in that snippets collection. + /// + public LsMagicMagic(IMagicSymbolResolver resolver) : base( + "lsmagic", + new Documentation + { + Summary = "Returns a list of all currently available magic commands." + }) + { + this.resolver = resolver; + } + + /// + public override ExecutionResult Run(string input, IChannel channel) => + // TODO: format as something nicer than a table. + resolver + .FindAllMagicSymbols() + .Select(magic => new + { + Name = magic.Name, + Documentation = magic.Documentation, + AssemblyName = magic.GetType().Assembly.GetName().Name + }) + .OrderBy(magic => magic.Name) + .ToList() + .ToExecutionResult(); + } +} diff --git a/src/Jupyter/Magic/Resolution/IMagicResolver.cs b/src/Jupyter/Magic/Resolution/IMagicResolver.cs new file mode 100644 index 0000000000..6a266a3d80 --- /dev/null +++ b/src/Jupyter/Magic/Resolution/IMagicResolver.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Jupyter.Core; +using Microsoft.Quantum.IQSharp.Common; + +using Newtonsoft.Json; + +namespace Microsoft.Quantum.IQSharp.Jupyter +{ + /// + /// Subinterface of + /// with additional functionality for discovering magic symbols. + /// + public interface IMagicSymbolResolver : ISymbolResolver + { + ISymbol ISymbolResolver.Resolve(string symbolName) => + this.Resolve(symbolName); + public new MagicSymbol Resolve(string symbolName); + + public IEnumerable FindAllMagicSymbols(); + } +} diff --git a/src/Jupyter/MagicResolver.cs b/src/Jupyter/Magic/Resolution/MagicResolver.cs similarity index 91% rename from src/Jupyter/MagicResolver.cs rename to src/Jupyter/Magic/Resolution/MagicResolver.cs index 7d30e7d0a4..3335c59dec 100644 --- a/src/Jupyter/MagicResolver.cs +++ b/src/Jupyter/Magic/Resolution/MagicResolver.cs @@ -20,7 +20,7 @@ namespace Microsoft.Quantum.IQSharp.Jupyter /// and all the Assemblies in global references (including those /// added via nuget Packages). /// - public class MagicSymbolResolver : ISymbolResolver + public class MagicSymbolResolver : IMagicSymbolResolver { private AssemblyInfo kernelAssembly; private Dictionary cache; @@ -33,12 +33,12 @@ public class MagicSymbolResolver : ISymbolResolver /// services to search assembly references for subclasses of /// . /// - public MagicSymbolResolver(IServiceProvider services, ILogger logger) + public MagicSymbolResolver(IServiceProvider services, ILogger logger) { this.cache = new Dictionary(); this.logger = logger; - this.kernelAssembly = new AssemblyInfo(typeof(IQSharpEngine).Assembly); + this.kernelAssembly = new AssemblyInfo(typeof(MagicSymbolResolver).Assembly); this.services = services; this.references = services.GetService(); } @@ -72,13 +72,13 @@ private IEnumerable RelevantAssemblies() /// Symbol names without a dot are resolved to the first symbol /// whose base name matches the given name. /// - public ISymbol Resolve(string symbolName) + public MagicSymbol Resolve(string symbolName) { if (symbolName == null || !symbolName.TrimStart().StartsWith("%")) return null; this.logger.LogDebug($"Looking for magic {symbolName}"); - foreach (var magic in RelevantAssemblies().SelectMany(FindMagic)) + foreach (var magic in FindAllMagicSymbols()) { if (symbolName.StartsWith(magic.Name)) { @@ -134,5 +134,9 @@ public IEnumerable FindMagic(AssemblyInfo assm) return result; } + + /// + public IEnumerable FindAllMagicSymbols() => + RelevantAssemblies().SelectMany(FindMagic); } } diff --git a/src/Python/qsharp/packages.py b/src/Python/qsharp/packages.py index 6d8537e06b..72a6ac7272 100644 --- a/src/Python/qsharp/packages.py +++ b/src/Python/qsharp/packages.py @@ -43,6 +43,6 @@ def add(self, package_name : str) -> None: session, downloading the package from NuGet.org or any other configured feeds as necessary. """ - logger.info("Loading package: " + package_name) - pkgs=self._client.add_package(package_name) + logger.info(f"Loading package: {package_name}") + pkgs = self._client.add_package(package_name) logger.info("Loading complete: " + ';'.join(str(e) for e in pkgs)) From 4fee88609421e9a5a29e7adc53dc0b6d0afa3e74 Mon Sep 17 00:00:00 2001 From: Christopher Granade Date: Tue, 7 Apr 2020 13:48:24 -0700 Subject: [PATCH 02/17] Slight improvements to header formatting. --- build/docs/build_docs.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/build/docs/build_docs.py b/build/docs/build_docs.py index 977760437f..214b009345 100644 --- a/build/docs/build_docs.py +++ b/build/docs/build_docs.py @@ -8,9 +8,12 @@ import click try: - import ruamel.yaml as yaml + from ruamel.yaml import YAML except ImportError: - import ruamel_yaml as yaml + from ruamel_yaml import YAML + +yaml = YAML() +yaml.indent(mapping=2, sequence=2) @click.command() @click.argument("OUTPUT_DIR") @@ -102,7 +105,7 @@ def format_as_document(magic, uid_base : str) -> str: return cleanup_markdown(f""" --- -{metadata_as_yaml.getvalue()} +{metadata_as_yaml.getvalue().rstrip()} --- + {header} {summary} {description} diff --git a/build/pack.ps1 b/build/pack.ps1 index 29d5ea7606..fc8885e317 100644 --- a/build/pack.ps1 +++ b/build/pack.ps1 @@ -133,4 +133,15 @@ if ($Env:ENABLE_DOCKER -eq "false") { } else { Write-Host "##[info]Packing Docker image..." Pack-Image -RepoName "iqsharp-base" -Dockerfile '../images/iqsharp-base/Dockerfile' -} \ No newline at end of file +} + +if (($Env:ENABLE_DOCKER -eq "true") -and ($Env:ENABLE_PYTHON -eq "true")) { + # If we can, pack docs using the documentation build container. + # We use the trick at https://blog.ropnop.com/plundering-docker-images/#extracting-files + # to build a new image containing all the docs we care about, then `docker cp` + # them out. + $tempTag = New-Guid | Select-Object -ExpandProperty Guid; + docker build -t $tempTag (Join-Path $PSScriptRoot "docs"); + $tempContainer = docker create $tempTag; + docker cp "${tempContainer}:/workdir/drops/docs/iqsharp-magic" (Join-Path $Env:DOCS_OUTDIR "iqsharp-magic") +} From df3ae0126d8fd2d93be977100621977ef0ac7051 Mon Sep 17 00:00:00 2001 From: Christopher Granade Date: Tue, 7 Apr 2020 14:34:16 -0700 Subject: [PATCH 04/17] Clarify pack.ps1 logic some. --- build/pack.ps1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build/pack.ps1 b/build/pack.ps1 index fc8885e317..ee163aef4a 100644 --- a/build/pack.ps1 +++ b/build/pack.ps1 @@ -135,7 +135,9 @@ if ($Env:ENABLE_DOCKER -eq "false") { Pack-Image -RepoName "iqsharp-base" -Dockerfile '../images/iqsharp-base/Dockerfile' } -if (($Env:ENABLE_DOCKER -eq "true") -and ($Env:ENABLE_PYTHON -eq "true")) { +if (($Env:ENABLE_DOCKER -eq "false") -or ($Env:ENABLE_PYTHON -eq "false")) {\ + Write-Host "##vso[task.logissue type=warning;]Skipping IQ# magic command documentation, either ENABLE_DOCKER or ENABLE_PYTHON was false."; +} else { # If we can, pack docs using the documentation build container. # We use the trick at https://blog.ropnop.com/plundering-docker-images/#extracting-files # to build a new image containing all the docs we care about, then `docker cp` From 71b2333471a2233ad8f64f6fcc2c4ae3dc08d434 Mon Sep 17 00:00:00 2001 From: Christopher Granade Date: Tue, 7 Apr 2020 14:47:58 -0700 Subject: [PATCH 05/17] Changed Dockerfile for CI to be stdin. --- build/docs/Dockerfile | 14 -------------- build/pack.ps1 | 21 ++++++++++++++++++++- 2 files changed, 20 insertions(+), 15 deletions(-) delete mode 100644 build/docs/Dockerfile diff --git a/build/docs/Dockerfile b/build/docs/Dockerfile deleted file mode 100644 index 508d19a6d4..0000000000 --- a/build/docs/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM mcr.microsoft.com/quantum/iqsharp-base:latest - -USER root -RUN pip install click ruamel.yaml -WORKDIR /workdir -RUN chown -R ${USER} /workdir - -USER ${USER} -ARG EXTRA_PACKAGES= -COPY build_docs.py /workdir -RUN python build_docs.py \ - /workdir/drops/docs/iqsharp-magic \ - microsoft.quantum.iqsharp.magic-ref \ - ${EXTRA_PACKAGES} diff --git a/build/pack.ps1 b/build/pack.ps1 index ee163aef4a..08c842a529 100644 --- a/build/pack.ps1 +++ b/build/pack.ps1 @@ -143,7 +143,26 @@ if (($Env:ENABLE_DOCKER -eq "false") -or ($Env:ENABLE_PYTHON -eq "false")) {\ # to build a new image containing all the docs we care about, then `docker cp` # them out. $tempTag = New-Guid | Select-Object -ExpandProperty Guid; - docker build -t $tempTag (Join-Path $PSScriptRoot "docs"); + # Note that we want to use a Dockerfile read from stdin so that we can more + # easily inject the right base image into the FROM line. In doing so, + # the build context should include the build_docs.py script that we need. + $dockerfile = @" +FROM ${Env:DOCKER_PREFIX}iqsharp-base:${Env:BUILD_BUILDNUMBER} + +USER root +RUN pip install click ruamel.yaml +WORKDIR /workdir +RUN chown -R `${USER} /workdir + +USER `${USER} +ARG EXTRA_PACKAGES= +COPY build_docs.py /workdir +RUN python build_docs.py \ + /workdir/drops/docs/iqsharp-magic \ + microsoft.quantum.iqsharp.magic-ref \ + `${EXTRA_PACKAGES} +"@; + $dockerfile | docker build -t $tempTag -f - (Join-Path $PSScriptRoot "docs"); $tempContainer = docker create $tempTag; docker cp "${tempContainer}:/workdir/drops/docs/iqsharp-magic" (Join-Path $Env:DOCS_OUTDIR "iqsharp-magic") } From 2993d2334a7a5ce08b63f716b7042b3fb1698c93 Mon Sep 17 00:00:00 2001 From: Christopher Granade Date: Tue, 7 Apr 2020 15:36:19 -0700 Subject: [PATCH 06/17] Update to 0.11 to try and fix build issues. --- build/ci.yml | 2 +- src/Tool/appsettings.json | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/build/ci.yml b/build/ci.yml index 76343709d3..3bb8115793 100644 --- a/build/ci.yml +++ b/build/ci.yml @@ -4,7 +4,7 @@ trigger: variables: Build.Major: 0 - Build.Minor: 10 + Build.Minor: 11 Drops.Dir: $(Build.ArtifactStagingDirectory)/drops jobs: diff --git a/src/Tool/appsettings.json b/src/Tool/appsettings.json index 68275089e3..f548296dd0 100644 --- a/src/Tool/appsettings.json +++ b/src/Tool/appsettings.json @@ -6,18 +6,18 @@ }, "AllowedHosts": "*", "DefaultPackageVersions": [ - "Microsoft.Quantum.Compiler::0.10.2003.1102-beta", + "Microsoft.Quantum.Compiler::0.11.2003.3107", - "Microsoft.Quantum.CsharpGeneration::0.10.2003.1102-beta", - "Microsoft.Quantum.Development.Kit::0.10.2003.1102-beta", - "Microsoft.Quantum.Simulators::0.10.2003.1102-beta", - "Microsoft.Quantum.Xunit::0.10.2003.1102-beta", + "Microsoft.Quantum.CsharpGeneration::0.11.2003.3107", + "Microsoft.Quantum.Development.Kit::0.11.2003.3107", + "Microsoft.Quantum.Simulators::0.11.2003.3107", + "Microsoft.Quantum.Xunit::0.11.2003.3107", - "Microsoft.Quantum.Standard::0.10.2003.1102-beta", - "Microsoft.Quantum.Chemistry::0.10.2003.1102-beta", - "Microsoft.Quantum.Chemistry.Jupyter::0.10.2003.1102-beta", - "Microsoft.Quantum.Numerics::0.10.2003.1102-beta", + "Microsoft.Quantum.Standard::0.11.2003.3107", + "Microsoft.Quantum.Chemistry::0.11.2003.3107", + "Microsoft.Quantum.Chemistry.Jupyter::0.11.2003.3107", + "Microsoft.Quantum.Numerics::0.11.2003.3107", - "Microsoft.Quantum.Research::0.10.2003.1102-beta" + "Microsoft.Quantum.Research::0.11.2003.3107" ] } From 4b8a802d4c0a240cbc8c73c84b1537991a7f2e82 Mon Sep 17 00:00:00 2001 From: Christopher Granade Date: Tue, 7 Apr 2020 15:53:54 -0700 Subject: [PATCH 07/17] Added string extension for stripping indents. --- src/Jupyter/Extensions.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Jupyter/Extensions.cs b/src/Jupyter/Extensions.cs index f202ec4383..84f31f1709 100644 --- a/src/Jupyter/Extensions.cs +++ b/src/Jupyter/Extensions.cs @@ -4,7 +4,7 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; -using System.Text; +using System.Text.RegularExpressions; using Microsoft.Extensions.DependencyInjection; using Microsoft.Jupyter.Core; using Microsoft.Jupyter.Core.Protocol; @@ -170,5 +170,9 @@ public static T WithStackTraceDisplay(this T simulator, IChannel channel) }; return simulator; } + + internal static string TrimLeadingWhitespace(this string s) => + new Regex(@"^\s+", RegexOptions.Multiline) + .Replace(s, string.Empty); } } From 1cd032516ceaf9cf85cd43ce20b4b8191dc1266e Mon Sep 17 00:00:00 2001 From: Christopher Granade Date: Tue, 7 Apr 2020 17:15:55 -0700 Subject: [PATCH 08/17] Revert changes to appsettings.json. --- src/Tool/appsettings.json | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/Tool/appsettings.json b/src/Tool/appsettings.json index f548296dd0..68275089e3 100644 --- a/src/Tool/appsettings.json +++ b/src/Tool/appsettings.json @@ -6,18 +6,18 @@ }, "AllowedHosts": "*", "DefaultPackageVersions": [ - "Microsoft.Quantum.Compiler::0.11.2003.3107", + "Microsoft.Quantum.Compiler::0.10.2003.1102-beta", - "Microsoft.Quantum.CsharpGeneration::0.11.2003.3107", - "Microsoft.Quantum.Development.Kit::0.11.2003.3107", - "Microsoft.Quantum.Simulators::0.11.2003.3107", - "Microsoft.Quantum.Xunit::0.11.2003.3107", + "Microsoft.Quantum.CsharpGeneration::0.10.2003.1102-beta", + "Microsoft.Quantum.Development.Kit::0.10.2003.1102-beta", + "Microsoft.Quantum.Simulators::0.10.2003.1102-beta", + "Microsoft.Quantum.Xunit::0.10.2003.1102-beta", - "Microsoft.Quantum.Standard::0.11.2003.3107", - "Microsoft.Quantum.Chemistry::0.11.2003.3107", - "Microsoft.Quantum.Chemistry.Jupyter::0.11.2003.3107", - "Microsoft.Quantum.Numerics::0.11.2003.3107", + "Microsoft.Quantum.Standard::0.10.2003.1102-beta", + "Microsoft.Quantum.Chemistry::0.10.2003.1102-beta", + "Microsoft.Quantum.Chemistry.Jupyter::0.10.2003.1102-beta", + "Microsoft.Quantum.Numerics::0.10.2003.1102-beta", - "Microsoft.Quantum.Research::0.11.2003.3107" + "Microsoft.Quantum.Research::0.10.2003.1102-beta" ] } From 6f67295b669b3981d0f0ee9d25c0146b611eafa3 Mon Sep 17 00:00:00 2001 From: Christopher Granade Date: Tue, 7 Apr 2020 17:33:37 -0700 Subject: [PATCH 09/17] Allow conditionally pulling in additional packages. --- build/pack.ps1 | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/build/pack.ps1 b/build/pack.ps1 index 08c842a529..387b3b2c33 100644 --- a/build/pack.ps1 +++ b/build/pack.ps1 @@ -143,6 +143,13 @@ if (($Env:ENABLE_DOCKER -eq "false") -or ($Env:ENABLE_PYTHON -eq "false")) {\ # to build a new image containing all the docs we care about, then `docker cp` # them out. $tempTag = New-Guid | Select-Object -ExpandProperty Guid; + # When building in release mode, we also want to document additional + # packages that contribute IQ# magic commands. + if ("$Env:BUILD_RELEASETYPE" -eq "release") { + $extraPackages = "--package Microsoft.Quantum.Katas --package Microsoft.Quantum.Chemistry.Jupyter"; + } else { + $extraPackages = ""; + } # Note that we want to use a Dockerfile read from stdin so that we can more # easily inject the right base image into the FROM line. In doing so, # the build context should include the build_docs.py script that we need. @@ -155,12 +162,11 @@ WORKDIR /workdir RUN chown -R `${USER} /workdir USER `${USER} -ARG EXTRA_PACKAGES= COPY build_docs.py /workdir RUN python build_docs.py \ /workdir/drops/docs/iqsharp-magic \ microsoft.quantum.iqsharp.magic-ref \ - `${EXTRA_PACKAGES} + $extraPackages "@; $dockerfile | docker build -t $tempTag -f - (Join-Path $PSScriptRoot "docs"); $tempContainer = docker create $tempTag; From ecba006663254f824db72c0f2a1fdc425741b852 Mon Sep 17 00:00:00 2001 From: Christopher Granade Date: Tue, 7 Apr 2020 18:01:22 -0700 Subject: [PATCH 10/17] Write toc.yml and index.md as well. --- build/docs/build_docs.py | 90 +++++++++++++++++++++++++++++++++------- 1 file changed, 76 insertions(+), 14 deletions(-) diff --git a/build/docs/build_docs.py b/build/docs/build_docs.py index 6250313ea1..a37bd08f90 100644 --- a/build/docs/build_docs.py +++ b/build/docs/build_docs.py @@ -1,8 +1,9 @@ import json import datetime +import dataclasses from io import StringIO -from typing import List, Optional +from typing import List, Optional, Dict from pathlib import Path import click @@ -15,6 +16,14 @@ yaml = YAML() yaml.indent(mapping=2, sequence=2) +@dataclasses.dataclass +class MagicReferenceDocument: + content: str + name: str + safe_name: str + uid: str + summary: str + @click.command() @click.argument("OUTPUT_DIR") @click.argument("UID_BASE") @@ -32,10 +41,20 @@ def main(output_dir : str, uid_base : str, package : List[str]): print("Generating Markdown files...") magics = qsharp.client._execute(r"%lsmagic") + all_magics = {} for magic in magics: magic_doc = format_as_document(magic, uid_base) - with open(output_dir / f"{magic['Name'].replace('%', '')}.md", 'w', encoding='utf8') as f: - f.write(magic_doc) + all_magics[magic_doc.name] = magic_doc + with open(output_dir / f"{magic_doc.safe_name}.md", 'w', encoding='utf8') as f: + f.write(magic_doc.content) + + toc_content = format_toc(all_magics) + with open(output_dir / "toc.yml", 'w', encoding='utf8') as f: + f.write(toc_content) + + index_content = format_index(all_magics, uid_base) + with open(output_dir / "index.md", 'w', encoding='utf8') as f: + f.write(index_content) def format_as_section(name : str, content : Optional[str]) -> str: content = content.strip() if content else None @@ -68,20 +87,30 @@ def cleanup_markdown(content : str): _cleanup_markdown(content) ).strip() + "\n" -def format_as_document(magic, uid_base : str) -> str: +def as_yaml_header(metadata) -> str: + # Convert the metadata header to YAML. + metadata_as_yaml = StringIO() + yaml.dump(metadata, metadata_as_yaml) + + return f"---\n{metadata_as_yaml.getvalue().rstrip()}\n---""" + +def format_as_document(magic, uid_base : str) -> MagicReferenceDocument: # NB: this function supports both the old and new Documentation format. # See https://github.com/microsoft/jupyter-core/pull/49. magic_name = magic['Name'].strip() + safe_name = magic_name.replace('%', '') + uid = f"{uid_base}.{safe_name}" metadata = { 'title': f"{magic_name} (magic command)", - 'uid': f"{uid_base}.{magic_name.replace('%', '')}", + 'uid': uid, 'ms.date': datetime.date.today().isoformat(), 'ms.topic': 'article' } header = f"# `{magic_name}`" doc = magic['Documentation'] - summary = format_as_section('Summary', doc.get('Summary', "")) + raw_summary = doc.get('Summary', "") + summary = format_as_section('Summary', raw_summary) description = format_as_section( 'Description', doc.get('Description', doc.get('Full', '')) @@ -99,14 +128,9 @@ def format_as_document(magic, uid_base : str) -> str: ) ) - # Convert the metadata header to YAML. - metadata_as_yaml = StringIO() - yaml.dump(metadata, metadata_as_yaml) - - return cleanup_markdown(f""" ---- -{metadata_as_yaml.getvalue().rstrip()} ---- + return MagicReferenceDocument( + content=cleanup_markdown(f""" +{as_yaml_header(metadata)}