Build magic docs

build/docs/README.md

@@ -0,0 +1,4 @@
+# Building IQ# Reference Documentation
+
+The contents of this folder automatically builds documentation for each
+available magic command.

build/docs/build_docs.py

@@ -0,0 +1,212 @@
+#!/bin/env python
+# -*- coding: utf-8 -*-
+##
+# build_docs.py: Builds documentation for IQ# magic commands as
+#     DocFX-compatible Markdown and YAML files.
+##
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+##
+
+"""
+Builds documentation for IQ# magic commands as
+DocFX-compatible Markdown and YAML files.
+
+Note that this command requires the IQ# kernel and the qsharp package to
+be installed, as well as Python 3.7 or later, click, and ruamel.yaml.
+"""
+
+import json
+import datetime
+import dataclasses
+
+from io import StringIO
+from typing import List, Optional, Dict
+from pathlib import Path
+
+import click
+
+try:
+    from ruamel.yaml import YAML
+except ImportError:
+    from ruamel_yaml import YAML
+
+yaml = YAML()
+yaml.indent(mapping=2, sequence=2)
+
+@dataclasses.dataclass
+class MagicReferenceDocument:
+    content: str
+    name: str
+    safe_name: str
+    uid: str
+    summary: str
+
+@click.command()
+@click.argument("OUTPUT_DIR")
+@click.argument("UID_BASE")
+@click.option("--package", "-p", multiple=True)
+def main(output_dir : str, uid_base : str, package : List[str]):
+    output_dir = Path(output_dir)
+    # Make the output directory if it doesn't already exist.
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    import qsharp
+
+    print("Adding packages...")
+    for package_name in package:
+        qsharp.packages.add(package_name)
+
+    print("Generating Markdown files...")
+    magics = qsharp.client._execute(r"%lsmagic")
+    all_magics = {}
+    for magic in magics:
+        magic_doc = format_as_document(magic, uid_base)
+        all_magics[magic_doc.name] = magic_doc
+        with open(output_dir / f"{magic_doc.safe_name}.md", 'w', encoding='utf8') as f:
+            f.write(magic_doc.content)
+
+    toc_content = format_toc(all_magics)
+    with open(output_dir / "toc.yml", 'w', encoding='utf8') as f:
+        f.write(toc_content)
+
+    index_content = format_index(all_magics, uid_base)
+    with open(output_dir / "index.md", 'w', encoding='utf8') as f:
+        f.write(index_content)
+
+def format_as_section(name : str, content : Optional[str]) -> str:
+    content = content.strip() if content else None
+    return f"""
+
+## {name}
+
+{content}
+
+""" if content else ""
+
+def _cleanup_markdown(content : str):
+    # Ensure that there is exactly one trailing newline, and that each line
+    # is free of trailing whitespace (with an exception for exactly two
+    # trailing spaces). 
+    # We also want to make sure that there are not two or more blank lines in
+    # a row.
+    prev_blank = False
+    for line in content.split("\n"):
+        cleaned_line = line.rstrip() if line != line.rstrip() + "  " else line
+        this_blank = cleaned_line == ""
+
+        if not (prev_blank and this_blank):
+            yield cleaned_line
+
+        prev_blank = this_blank
+
+def cleanup_markdown(content : str):
+    return "\n".join(
+        _cleanup_markdown(content)
+    ).strip() + "\n"
+
+def as_yaml_header(metadata) -> str:
+    # Convert the metadata header to YAML.
+    metadata_as_yaml = StringIO()
+    yaml.dump(metadata, metadata_as_yaml)
+
+    return f"---\n{metadata_as_yaml.getvalue().rstrip()}\n---"""
+
+def format_as_document(magic, uid_base : str) -> MagicReferenceDocument:
+    # NB: this function supports both the old and new Documentation format.
+    #     See https://github.com/microsoft/jupyter-core/pull/49.
+    magic_name = magic['Name'].strip()
+    safe_name = magic_name.replace('%', '')
+    uid = f"{uid_base}.{safe_name}"
+    metadata = {
+        'title': f"{magic_name} (magic command)",
+        'uid': uid,
+        'ms.date': datetime.date.today().isoformat(),
+        'ms.topic': 'article'
+    }
+    header = f"# `{magic_name}`"
+    doc = magic['Documentation']
+
+    raw_summary = doc.get('Summary', "")
+    summary = format_as_section('Summary', raw_summary)
+    description = format_as_section(
+        'Description',
+        doc.get('Description', doc.get('Full', ''))
+    )
+    remarks = format_as_section('Remarks', doc.get('Remarks', ""))
+
+    raw_examples = doc.get('Examples', [])
+    examples = "\n".join(
+        format_as_section("Example", example)
+        for example in raw_examples
+    ) if raw_examples else ""
+
+    raw_see_also = doc.get('SeeAlso', [])
+    see_also = format_as_section(
+        "See Also",
+        "\n".join(
+            f"- [{description}]({target})"
+            for description, target in raw_see_also
+        )
+    ) if raw_see_also else ""
+
+    return MagicReferenceDocument(
+        content=cleanup_markdown(f"""
+{as_yaml_header(metadata)}
+
+<!--
+    NB: This file has been automatically generated from {magic.get("AssemblyName", "<unknown>")}.dll,
+        please do not manually edit it.
+
+    [DEBUG] JSON source:
+        {json.dumps(magic)}
+-->
+
+{header}
+{summary}
+{description}
+{remarks}
+{examples}
+{see_also}
+        """),
+        name=magic_name, safe_name=safe_name,
+        uid=uid,
+        summary=raw_summary
+    )
+
+def format_toc(all_magics : Dict[str, MagicReferenceDocument]) -> str:
+    toc_content = [
+        {
+            'href': f"{doc.safe_name}.md",
+            'name': f"{doc.name} magic command"
+        }
+        for magic_name, doc in sorted(all_magics.items(), key=lambda item: item[0])
+    ]
+
+    as_yaml = StringIO()
+    yaml.dump(toc_content, as_yaml)
+
+    return as_yaml.getvalue()
+
+
+def format_index(all_magics : Dict[str, MagicReferenceDocument], uid_base : str) -> str:
+    index_content = "\n".join(
+        f"| [`{magic_name}`](xref:{doc.uid}) | {doc.summary} |"
+        for magic_name, doc in sorted(all_magics.items(), key=lambda item: item[0])
+    )
+    metadata = {
+        'title': "IQ# Magic Commands",
+        'uid': f"{uid_base}.index",
+        'ms.date': datetime.date.today().isoformat(),
+        'ms.topic': 'article'
+    }
+    return cleanup_markdown(f"""
+# IQ# Magic Commands
+
+| Magic Command | Summary |
+|---------------|---------|
+{index_content}
+    """)
+
+if __name__ == "__main__":
+    main()

build/pack.ps1

@@ -62,12 +62,27 @@ function Pack-Image() {
         return
     }
 
+
+    <# If we are building a non-release build, we need to inject the
+       prerelease feed as well.
+       Note that since this will appear as an argument to docker build, which
+       then evaluates the build argument using Bash, we
+       need \" to be in the value of $extraNugetSources so that the final XML
+       contains just a ". Thus the correct escape sequence is \`".
+    #>
+    if ("$Env:BUILD_RELEASETYPE" -ne "release") {
+        $extraNugetSources = "<add key=\`"prerelease\`" value=\`"https://pkgs.dev.azure.com/ms-quantum-public/9af4e09e-a436-4aca-9559-2094cfe8d80c/_packaging/alpha%40Local/nuget/v3/index.json\`" />";
+    } else {
+        $extraNugetSources = "";
+    }
+
     docker build `
         <# We treat $DROP_DIR as the build context, as we will need to ADD
            nuget packages into the image. #> `
         $Env:DROPS_DIR `
         <# This means that the Dockerfile lives outside the build context. #> `
         -f (Join-Path $PSScriptRoot $Dockerfile) `
+        --build-arg EXTRA_NUGET_SOURCES="$extraNugetSources" `
         <# Next, we tell Docker what version of IQ# to install. #> `
         --build-arg IQSHARP_VERSION=$Env:NUGET_VERSION `
         <# Finally, we tag the image with the current build number. #> `
@@ -133,4 +148,42 @@ if ($Env:ENABLE_DOCKER -eq "false") {
 } else {
     Write-Host "##[info]Packing Docker image..."
     Pack-Image -RepoName "iqsharp-base" -Dockerfile '../images/iqsharp-base/Dockerfile'
-}
+}
+
+if (($Env:ENABLE_DOCKER -eq "false") -or ($Env:ENABLE_PYTHON -eq "false")) {\
+    Write-Host "##vso[task.logissue type=warning;]Skipping IQ# magic command documentation, either ENABLE_DOCKER or ENABLE_PYTHON was false.";
+} else {
+    # If we can, pack docs using the documentation build container.
+    # We use the trick at https://blog.ropnop.com/plundering-docker-images/#extracting-files
+    # to build a new image containing all the docs we care about, then `docker cp`
+    # them out.
+    $tempTag = New-Guid | Select-Object -ExpandProperty Guid;
+    # When building in release mode, we also want to document additional
+    # packages that contribute IQ# magic commands.
+    if ("$Env:BUILD_RELEASETYPE" -eq "release") {
+        $extraPackages = "--package Microsoft.Quantum.Katas --package Microsoft.Quantum.Chemistry.Jupyter";
+    } else {
+        $extraPackages = "";
+    }
+    # Note that we want to use a Dockerfile read from stdin so that we can more
+    # easily inject the right base image into the FROM line. In doing so,
+    # the build context should include the build_docs.py script that we need.
+    $dockerfile = @"
+FROM ${Env:DOCKER_PREFIX}iqsharp-base:${Env:BUILD_BUILDNUMBER}
+
+USER root
+RUN pip install click ruamel.yaml
+WORKDIR /workdir
+RUN chown -R `${USER} /workdir
+
+USER `${USER}
+COPY build_docs.py /workdir
+RUN python build_docs.py \
+        /workdir/drops/docs/iqsharp-magic \
+        microsoft.quantum.iqsharp.magic-ref \
+        $extraPackages
+"@;
+    $dockerfile | docker build -t $tempTag -f - (Join-Path $PSScriptRoot "docs");
+    $tempContainer = docker create $tempTag;
+    docker cp "${tempContainer}:/workdir/drops/docs/iqsharp-magic" (Join-Path $Env:DOCS_OUTDIR "iqsharp-magic")
+}

images/iqsharp-base/Dockerfile

@@ -76,16 +76,20 @@ WORKDIR ${HOME}
 
 # Provide local copies of all relevant packages.
 ENV LOCAL_PACKAGES=${HOME}/.packages
+ARG EXTRA_NUGET_SOURCES=
 # Add the local NuGet packages folder as a source.
 RUN mkdir -p ${HOME}/.nuget/NuGet && \
     echo "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n\
           <configuration>\n\
               <packageSources>\n\
                   <add key=\"nuget\" value=\"https://api.nuget.org/v3/index.json\" />\n\
                   <add key=\"context\" value=\"${LOCAL_PACKAGES}/nugets\" />\n\
+                  ${EXTRA_NUGET_SOURCES}\n\
               </packageSources>\n\
           </configuration>\n\
-    " > ${HOME}/.nuget/NuGet/NuGet.Config
+    " > ${HOME}/.nuget/NuGet/NuGet.Config && \
+    echo "Using NuGet.Config:" && \
+    cat ${HOME}/.nuget/NuGet/NuGet.Config
 # Add Python and NuGet packages from the build context
 ADD nugets/*.nupkg ${LOCAL_PACKAGES}/nugets/
 ADD wheels/*.whl ${LOCAL_PACKAGES}/wheels/

src/Jupyter/Extensions.cs

@@ -35,6 +35,7 @@ public static ChannelWithNewLines WithNewLines(this IChannel original) =>
         public static void AddIQSharpKernel(this IServiceCollection services)
         {
             services.AddSingleton<ISymbolResolver, Jupyter.SymbolResolver>();
+            services.AddSingleton<IMagicSymbolResolver, Jupyter.MagicSymbolResolver>();
             services.AddSingleton<IExecutionEngine, Jupyter.IQSharpEngine>();
             services.AddSingleton<IConfigurationSource, ConfigurationSource>();
         }

src/Jupyter/IQSharpEngine.cs

@@ -39,15 +39,16 @@ public IQSharpEngine(
             IConfigurationSource configurationSource,
             PerformanceMonitor performanceMonitor,
             IShellRouter shellRouter,
-            IEventService eventService
+            IEventService eventService,
+            IMagicSymbolResolver magicSymbolResolver
         ) : base(shell, shellRouter, context, logger, services)
         {
             this.performanceMonitor = performanceMonitor;
             performanceMonitor.Start();
 
             this.Snippets = services.GetService<ISnippets>();
             this.SymbolsResolver = services.GetService<ISymbolResolver>();
-            this.MagicResolver = new MagicSymbolResolver(services, logger);
+            this.MagicResolver = magicSymbolResolver;
 
             RegisterDisplayEncoder(new IQSharpSymbolToHtmlResultEncoder());
             RegisterDisplayEncoder(new IQSharpSymbolToTextResultEncoder());

src/Jupyter/Magic/LsMagicMagic.cs

@@ -0,0 +1,48 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System.Linq;
+
+using Microsoft.Jupyter.Core;
+using Microsoft.Quantum.IQSharp;
+using Microsoft.Quantum.IQSharp.Jupyter;
+
+namespace Microsoft.Quantum.IQSharp.Jupyter
+{
+    /// <summary>
+    ///     A magic command that lists what magic commands are currently
+    ///     available.
+    /// </summary>
+    public class LsMagicMagic : AbstractMagic
+    {
+        private readonly IMagicSymbolResolver resolver;
+        /// <summary>
+        ///     Given a given snippets collection, constructs a new magic command
+        ///     that queries callables defined in that snippets collection.
+        /// </summary>
+        public LsMagicMagic(IMagicSymbolResolver resolver) : base(
+            "lsmagic",
+            new Documentation
+            {
+                Summary = "Returns a list of all currently available magic commands."
+            })
+        {
+            this.resolver = resolver;
+        }
+
+        /// <inheritdoc />
+        public override ExecutionResult Run(string input, IChannel channel) =>
+            // TODO: format as something nicer than a table.
+            resolver
+                .FindAllMagicSymbols()
+                .Select(magic => new
+                {
+                    Name = magic.Name,
+                    Documentation = magic.Documentation,
+                    AssemblyName = magic.GetType().Assembly.GetName().Name
+                })
+                .OrderBy(magic => magic.Name)
+                .ToList()
+                .ToExecutionResult();
+    }
+}