diff --git a/build/docs/README.md b/build/docs/README.md new file mode 100644 index 0000000000..5819103327 --- /dev/null +++ b/build/docs/README.md @@ -0,0 +1,4 @@ +# Building IQ# Reference Documentation + +The contents of this folder automatically builds documentation for each +available magic command. diff --git a/build/docs/build_docs.py b/build/docs/build_docs.py new file mode 100644 index 0000000000..4dda43574d --- /dev/null +++ b/build/docs/build_docs.py @@ -0,0 +1,212 @@ +#!/bin/env python +# -*- coding: utf-8 -*- +## +# build_docs.py: Builds documentation for IQ# magic commands as +# DocFX-compatible Markdown and YAML files. +## +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +## + +""" +Builds documentation for IQ# magic commands as +DocFX-compatible Markdown and YAML files. + +Note that this command requires the IQ# kernel and the qsharp package to +be installed, as well as Python 3.7 or later, click, and ruamel.yaml. +""" + +import json +import datetime +import dataclasses + +from io import StringIO +from typing import List, Optional, Dict +from pathlib import Path + +import click + +try: + from ruamel.yaml import YAML +except ImportError: + from ruamel_yaml import YAML + +yaml = YAML() +yaml.indent(mapping=2, sequence=2) + +@dataclasses.dataclass +class MagicReferenceDocument: + content: str + name: str + safe_name: str + uid: str + summary: str + +@click.command() +@click.argument("OUTPUT_DIR") +@click.argument("UID_BASE") +@click.option("--package", "-p", multiple=True) +def main(output_dir : str, uid_base : str, package : List[str]): + output_dir = Path(output_dir) + # Make the output directory if it doesn't already exist. + output_dir.mkdir(parents=True, exist_ok=True) + + import qsharp + + print("Adding packages...") + for package_name in package: + qsharp.packages.add(package_name) + + print("Generating Markdown files...") + magics = qsharp.client._execute(r"%lsmagic") + all_magics = {} + for magic in magics: + magic_doc = format_as_document(magic, uid_base) + all_magics[magic_doc.name] = magic_doc + with open(output_dir / f"{magic_doc.safe_name}.md", 'w', encoding='utf8') as f: + f.write(magic_doc.content) + + toc_content = format_toc(all_magics) + with open(output_dir / "toc.yml", 'w', encoding='utf8') as f: + f.write(toc_content) + + index_content = format_index(all_magics, uid_base) + with open(output_dir / "index.md", 'w', encoding='utf8') as f: + f.write(index_content) + +def format_as_section(name : str, content : Optional[str]) -> str: + content = content.strip() if content else None + return f""" + +## {name} + +{content} + +""" if content else "" + +def _cleanup_markdown(content : str): + # Ensure that there is exactly one trailing newline, and that each line + # is free of trailing whitespace (with an exception for exactly two + # trailing spaces). + # We also want to make sure that there are not two or more blank lines in + # a row. + prev_blank = False + for line in content.split("\n"): + cleaned_line = line.rstrip() if line != line.rstrip() + " " else line + this_blank = cleaned_line == "" + + if not (prev_blank and this_blank): + yield cleaned_line + + prev_blank = this_blank + +def cleanup_markdown(content : str): + return "\n".join( + _cleanup_markdown(content) + ).strip() + "\n" + +def as_yaml_header(metadata) -> str: + # Convert the metadata header to YAML. + metadata_as_yaml = StringIO() + yaml.dump(metadata, metadata_as_yaml) + + return f"---\n{metadata_as_yaml.getvalue().rstrip()}\n---""" + +def format_as_document(magic, uid_base : str) -> MagicReferenceDocument: + # NB: this function supports both the old and new Documentation format. + # See https://github.com/microsoft/jupyter-core/pull/49. + magic_name = magic['Name'].strip() + safe_name = magic_name.replace('%', '') + uid = f"{uid_base}.{safe_name}" + metadata = { + 'title': f"{magic_name} (magic command)", + 'uid': uid, + 'ms.date': datetime.date.today().isoformat(), + 'ms.topic': 'article' + } + header = f"# `{magic_name}`" + doc = magic['Documentation'] + + raw_summary = doc.get('Summary', "") + summary = format_as_section('Summary', raw_summary) + description = format_as_section( + 'Description', + doc.get('Description', doc.get('Full', '')) + ) + remarks = format_as_section('Remarks', doc.get('Remarks', "")) + + raw_examples = doc.get('Examples', []) + examples = "\n".join( + format_as_section("Example", example) + for example in raw_examples + ) if raw_examples else "" + + raw_see_also = doc.get('SeeAlso', []) + see_also = format_as_section( + "See Also", + "\n".join( + f"- [{description}]({target})" + for description, target in raw_see_also + ) + ) if raw_see_also else "" + + return MagicReferenceDocument( + content=cleanup_markdown(f""" +{as_yaml_header(metadata)} + + + +{header} +{summary} +{description} +{remarks} +{examples} +{see_also} + """), + name=magic_name, safe_name=safe_name, + uid=uid, + summary=raw_summary + ) + +def format_toc(all_magics : Dict[str, MagicReferenceDocument]) -> str: + toc_content = [ + { + 'href': f"{doc.safe_name}.md", + 'name': f"{doc.name} magic command" + } + for magic_name, doc in sorted(all_magics.items(), key=lambda item: item[0]) + ] + + as_yaml = StringIO() + yaml.dump(toc_content, as_yaml) + + return as_yaml.getvalue() + + +def format_index(all_magics : Dict[str, MagicReferenceDocument], uid_base : str) -> str: + index_content = "\n".join( + f"| [`{magic_name}`](xref:{doc.uid}) | {doc.summary} |" + for magic_name, doc in sorted(all_magics.items(), key=lambda item: item[0]) + ) + metadata = { + 'title': "IQ# Magic Commands", + 'uid': f"{uid_base}.index", + 'ms.date': datetime.date.today().isoformat(), + 'ms.topic': 'article' + } + return cleanup_markdown(f""" +# IQ# Magic Commands + +| Magic Command | Summary | +|---------------|---------| +{index_content} + """) + +if __name__ == "__main__": + main() diff --git a/build/pack.ps1 b/build/pack.ps1 index 29d5ea7606..721288aea9 100644 --- a/build/pack.ps1 +++ b/build/pack.ps1 @@ -62,12 +62,27 @@ function Pack-Image() { return } + + <# If we are building a non-release build, we need to inject the + prerelease feed as well. + Note that since this will appear as an argument to docker build, which + then evaluates the build argument using Bash, we + need \" to be in the value of $extraNugetSources so that the final XML + contains just a ". Thus the correct escape sequence is \`". + #> + if ("$Env:BUILD_RELEASETYPE" -ne "release") { + $extraNugetSources = ""; + } else { + $extraNugetSources = ""; + } + docker build ` <# We treat $DROP_DIR as the build context, as we will need to ADD nuget packages into the image. #> ` $Env:DROPS_DIR ` <# This means that the Dockerfile lives outside the build context. #> ` -f (Join-Path $PSScriptRoot $Dockerfile) ` + --build-arg EXTRA_NUGET_SOURCES="$extraNugetSources" ` <# Next, we tell Docker what version of IQ# to install. #> ` --build-arg IQSHARP_VERSION=$Env:NUGET_VERSION ` <# Finally, we tag the image with the current build number. #> ` @@ -133,4 +148,42 @@ if ($Env:ENABLE_DOCKER -eq "false") { } else { Write-Host "##[info]Packing Docker image..." Pack-Image -RepoName "iqsharp-base" -Dockerfile '../images/iqsharp-base/Dockerfile' -} \ No newline at end of file +} + +if (($Env:ENABLE_DOCKER -eq "false") -or ($Env:ENABLE_PYTHON -eq "false")) {\ + Write-Host "##vso[task.logissue type=warning;]Skipping IQ# magic command documentation, either ENABLE_DOCKER or ENABLE_PYTHON was false."; +} else { + # If we can, pack docs using the documentation build container. + # We use the trick at https://blog.ropnop.com/plundering-docker-images/#extracting-files + # to build a new image containing all the docs we care about, then `docker cp` + # them out. + $tempTag = New-Guid | Select-Object -ExpandProperty Guid; + # When building in release mode, we also want to document additional + # packages that contribute IQ# magic commands. + if ("$Env:BUILD_RELEASETYPE" -eq "release") { + $extraPackages = "--package Microsoft.Quantum.Katas --package Microsoft.Quantum.Chemistry.Jupyter"; + } else { + $extraPackages = ""; + } + # Note that we want to use a Dockerfile read from stdin so that we can more + # easily inject the right base image into the FROM line. In doing so, + # the build context should include the build_docs.py script that we need. + $dockerfile = @" +FROM ${Env:DOCKER_PREFIX}iqsharp-base:${Env:BUILD_BUILDNUMBER} + +USER root +RUN pip install click ruamel.yaml +WORKDIR /workdir +RUN chown -R `${USER} /workdir + +USER `${USER} +COPY build_docs.py /workdir +RUN python build_docs.py \ + /workdir/drops/docs/iqsharp-magic \ + microsoft.quantum.iqsharp.magic-ref \ + $extraPackages +"@; + $dockerfile | docker build -t $tempTag -f - (Join-Path $PSScriptRoot "docs"); + $tempContainer = docker create $tempTag; + docker cp "${tempContainer}:/workdir/drops/docs/iqsharp-magic" (Join-Path $Env:DOCS_OUTDIR "iqsharp-magic") +} diff --git a/images/iqsharp-base/Dockerfile b/images/iqsharp-base/Dockerfile index be800763a1..269ba1216f 100644 --- a/images/iqsharp-base/Dockerfile +++ b/images/iqsharp-base/Dockerfile @@ -76,6 +76,7 @@ WORKDIR ${HOME} # Provide local copies of all relevant packages. ENV LOCAL_PACKAGES=${HOME}/.packages +ARG EXTRA_NUGET_SOURCES= # Add the local NuGet packages folder as a source. RUN mkdir -p ${HOME}/.nuget/NuGet && \ echo "\n\ @@ -83,9 +84,12 @@ RUN mkdir -p ${HOME}/.nuget/NuGet && \ \n\ \n\ \n\ + ${EXTRA_NUGET_SOURCES}\n\ \n\ \n\ - " > ${HOME}/.nuget/NuGet/NuGet.Config + " > ${HOME}/.nuget/NuGet/NuGet.Config && \ + echo "Using NuGet.Config:" && \ + cat ${HOME}/.nuget/NuGet/NuGet.Config # Add Python and NuGet packages from the build context ADD nugets/*.nupkg ${LOCAL_PACKAGES}/nugets/ ADD wheels/*.whl ${LOCAL_PACKAGES}/wheels/ diff --git a/src/Jupyter/Extensions.cs b/src/Jupyter/Extensions.cs index a80b941c41..a4ccf50cd6 100644 --- a/src/Jupyter/Extensions.cs +++ b/src/Jupyter/Extensions.cs @@ -35,6 +35,7 @@ public static ChannelWithNewLines WithNewLines(this IChannel original) => public static void AddIQSharpKernel(this IServiceCollection services) { services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); } diff --git a/src/Jupyter/IQSharpEngine.cs b/src/Jupyter/IQSharpEngine.cs index a275044455..122284b7d6 100644 --- a/src/Jupyter/IQSharpEngine.cs +++ b/src/Jupyter/IQSharpEngine.cs @@ -39,7 +39,8 @@ public IQSharpEngine( IConfigurationSource configurationSource, PerformanceMonitor performanceMonitor, IShellRouter shellRouter, - IEventService eventService + IEventService eventService, + IMagicSymbolResolver magicSymbolResolver ) : base(shell, shellRouter, context, logger, services) { this.performanceMonitor = performanceMonitor; @@ -47,7 +48,7 @@ IEventService eventService this.Snippets = services.GetService(); this.SymbolsResolver = services.GetService(); - this.MagicResolver = new MagicSymbolResolver(services, logger); + this.MagicResolver = magicSymbolResolver; RegisterDisplayEncoder(new IQSharpSymbolToHtmlResultEncoder()); RegisterDisplayEncoder(new IQSharpSymbolToTextResultEncoder()); diff --git a/src/Jupyter/Magic/LsMagicMagic.cs b/src/Jupyter/Magic/LsMagicMagic.cs new file mode 100644 index 0000000000..1306bb6a08 --- /dev/null +++ b/src/Jupyter/Magic/LsMagicMagic.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Linq; + +using Microsoft.Jupyter.Core; +using Microsoft.Quantum.IQSharp; +using Microsoft.Quantum.IQSharp.Jupyter; + +namespace Microsoft.Quantum.IQSharp.Jupyter +{ + /// + /// A magic command that lists what magic commands are currently + /// available. + /// + public class LsMagicMagic : AbstractMagic + { + private readonly IMagicSymbolResolver resolver; + /// + /// Given a given snippets collection, constructs a new magic command + /// that queries callables defined in that snippets collection. + /// + public LsMagicMagic(IMagicSymbolResolver resolver) : base( + "lsmagic", + new Documentation + { + Summary = "Returns a list of all currently available magic commands." + }) + { + this.resolver = resolver; + } + + /// + public override ExecutionResult Run(string input, IChannel channel) => + // TODO: format as something nicer than a table. + resolver + .FindAllMagicSymbols() + .Select(magic => new + { + Name = magic.Name, + Documentation = magic.Documentation, + AssemblyName = magic.GetType().Assembly.GetName().Name + }) + .OrderBy(magic => magic.Name) + .ToList() + .ToExecutionResult(); + } +} diff --git a/src/Jupyter/Magic/Resolution/IMagicResolver.cs b/src/Jupyter/Magic/Resolution/IMagicResolver.cs new file mode 100644 index 0000000000..6a266a3d80 --- /dev/null +++ b/src/Jupyter/Magic/Resolution/IMagicResolver.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Jupyter.Core; +using Microsoft.Quantum.IQSharp.Common; + +using Newtonsoft.Json; + +namespace Microsoft.Quantum.IQSharp.Jupyter +{ + /// + /// Subinterface of + /// with additional functionality for discovering magic symbols. + /// + public interface IMagicSymbolResolver : ISymbolResolver + { + ISymbol ISymbolResolver.Resolve(string symbolName) => + this.Resolve(symbolName); + public new MagicSymbol Resolve(string symbolName); + + public IEnumerable FindAllMagicSymbols(); + } +} diff --git a/src/Jupyter/MagicResolver.cs b/src/Jupyter/Magic/Resolution/MagicResolver.cs similarity index 91% rename from src/Jupyter/MagicResolver.cs rename to src/Jupyter/Magic/Resolution/MagicResolver.cs index 7d30e7d0a4..3335c59dec 100644 --- a/src/Jupyter/MagicResolver.cs +++ b/src/Jupyter/Magic/Resolution/MagicResolver.cs @@ -20,7 +20,7 @@ namespace Microsoft.Quantum.IQSharp.Jupyter /// and all the Assemblies in global references (including those /// added via nuget Packages). /// - public class MagicSymbolResolver : ISymbolResolver + public class MagicSymbolResolver : IMagicSymbolResolver { private AssemblyInfo kernelAssembly; private Dictionary cache; @@ -33,12 +33,12 @@ public class MagicSymbolResolver : ISymbolResolver /// services to search assembly references for subclasses of /// . /// - public MagicSymbolResolver(IServiceProvider services, ILogger logger) + public MagicSymbolResolver(IServiceProvider services, ILogger logger) { this.cache = new Dictionary(); this.logger = logger; - this.kernelAssembly = new AssemblyInfo(typeof(IQSharpEngine).Assembly); + this.kernelAssembly = new AssemblyInfo(typeof(MagicSymbolResolver).Assembly); this.services = services; this.references = services.GetService(); } @@ -72,13 +72,13 @@ private IEnumerable RelevantAssemblies() /// Symbol names without a dot are resolved to the first symbol /// whose base name matches the given name. /// - public ISymbol Resolve(string symbolName) + public MagicSymbol Resolve(string symbolName) { if (symbolName == null || !symbolName.TrimStart().StartsWith("%")) return null; this.logger.LogDebug($"Looking for magic {symbolName}"); - foreach (var magic in RelevantAssemblies().SelectMany(FindMagic)) + foreach (var magic in FindAllMagicSymbols()) { if (symbolName.StartsWith(magic.Name)) { @@ -134,5 +134,9 @@ public IEnumerable FindMagic(AssemblyInfo assm) return result; } + + /// + public IEnumerable FindAllMagicSymbols() => + RelevantAssemblies().SelectMany(FindMagic); } } diff --git a/src/Python/qsharp/packages.py b/src/Python/qsharp/packages.py index 6d8537e06b..72a6ac7272 100644 --- a/src/Python/qsharp/packages.py +++ b/src/Python/qsharp/packages.py @@ -43,6 +43,6 @@ def add(self, package_name : str) -> None: session, downloading the package from NuGet.org or any other configured feeds as necessary. """ - logger.info("Loading package: " + package_name) - pkgs=self._client.add_package(package_name) + logger.info(f"Loading package: {package_name}") + pkgs = self._client.add_package(package_name) logger.info("Loading complete: " + ';'.join(str(e) for e in pkgs))