From f46f8e250fad6a6a83de4319ea0bcab083ce25b9 Mon Sep 17 00:00:00 2001 From: Eric Lunderberg Date: Thu, 12 Aug 2021 09:08:15 -0500 Subject: [PATCH 1/4] [Docs] Moved the generated tutorials folders into a _staging folder. Previously, reorganization or renaming of tutorials could cause documentation tests to fail in CI. The CI checks out the version to be tested, which may still have generated documents in `docs/tutorials` and `docs/vta/tutorials`. If a PR moves these to different folders, then they show up as duplicate `*.rst` files, resulting in sphinx warnings. This commit makes a `docs/_staging` folder in which sphinx is run. All tutorials are generated within this folder, and the entire folder can be deleted with `make clean`. As a result, it is safe to reorganize the tutorial without impacting CI. --- .gitignore | 2 +- docs/Makefile | 132 +++++++++++++++++++++++++++++++------------------- docs/conf.py | 54 +++++++++++++-------- 3 files changed, 115 insertions(+), 73 deletions(-) diff --git a/.gitignore b/.gitignore index e24999ef0d5c..7141aaeb192f 100644 --- a/.gitignore +++ b/.gitignore @@ -63,7 +63,7 @@ instance/ # Sphinx documentation docs/_build/ -docs/gen_modules +docs/_staging/ # PyBuilder /target/ diff --git a/docs/Makefile b/docs/Makefile index ca4f6e9a08f0..1922d056a130 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -23,15 +23,16 @@ SPHINXOPTS = SPHINXBUILD = python3 -m sphinx PAPER = BUILDDIR = _build +STAGINGDIR = _staging # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +ALLSPHINXOPTS = -d $(PWD)/$(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext staging help: @echo "Please use \`make ' where is one of" @@ -61,44 +62,73 @@ help: @echo " coverage to run coverage check of the documentation (if enabled)" clean: - rm -rf $(BUILDDIR)/* + rm -rf $(BUILDDIR) + rm -rf $(STAGINGDIR) + # Remove folders that have since been relocated into + # $(STAGINGDIR). This allows `task_sphinx_precheck.sh` to + # run, even if a commit that predates $(STAGINGDIR) was + # previously run on that node. rm -rf gen_modules + rm -rf user_tutorials rm -rf tutorials rm -rf vta/tutorials -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html +staging: + mkdir -p $(STAGINGDIR) + + # Remove any symlinks that currently exist + find $(STAGINGDIR) -type l -exec rm {} \; + + # Reproduce the directory structure + find . \ + -path ./$(BUILDDIR) -prune -o -path ./$(STAGINGDIR) -prune -o \ + -name "*.rst" \ + -printf "$(STAGINGDIR)/%h\n" \ + | sort | uniq | xargs mkdir -p + + # Symlink all .rst files into the staging directory + find . \ + -path ./$(BUILDDIR) -prune -o -path ./$(STAGINGDIR) -prune -o \ + -name "*.rst" \ + -exec ln -s $(PWD)/{} $(STAGINGDIR)/{} \; + + ln -s $(PWD)/conf.py $(STAGINGDIR)/conf.py + ln -s $(PWD)/_static $(STAGINGDIR)/_static + + +html: staging + cd $(STAGINGDIR) && $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(PWD)/$(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml +dirhtml: staging + cd $(STAGINGDIR) && $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(PWD)/$(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml +singlehtml: staging + cd $(STAGINGDIR) && $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(PWD)/$(BUILDDIR)/singlehtml @echo @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle +pickle: staging + cd $(STAGINGDIR) && $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(PWD)/$(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json +json: staging + cd $(STAGINGDIR) && $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(PWD)/$(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp +htmlhelp: staging + cd $(STAGINGDIR) && $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(PWD)/$(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp +qthelp: staging + cd $(STAGINGDIR) && $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(PWD)/$(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @@ -106,16 +136,16 @@ qthelp: @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/rabit.qhc" -applehelp: - $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp +applehelp: staging + cd $(STAGINGDIR) && $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(PWD)/$(BUILDDIR)/applehelp @echo @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." @echo "N.B. You won't be able to view it unless you put it in" \ "~/Library/Documentation/Help or install it in your application" \ "bundle." -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp +devhelp: staging + cd $(STAGINGDIR) && $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(PWD)/$(BUILDDIR)/devhelp @echo @echo "Build finished." @echo "To view the help file:" @@ -123,85 +153,85 @@ devhelp: @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/rabit" @echo "# devhelp" -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub +epub: staging + cd $(STAGINGDIR) && $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(PWD)/$(BUILDDIR)/epub @echo @echo "Build finished. The epub file is in $(BUILDDIR)/epub." -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex +latex: staging + cd $(STAGINGDIR) && $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(PWD)/$(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make' in that directory to run these through (pdf)latex" \ "(use \`make latexpdf' here to do that automatically)." -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex +latexpdf: staging + cd $(STAGINGDIR) && $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(PWD)/$(BUILDDIR)/latex @echo "Running LaTeX files through pdflatex..." $(MAKE) -C $(BUILDDIR)/latex all-pdf @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." -latexpdfja: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex +latexpdfja: staging + cd $(STAGINGDIR) && $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(PWD)/$(BUILDDIR)/latex @echo "Running LaTeX files through platex and dvipdfmx..." $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text +text: staging + cd $(STAGINGDIR) && $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(PWD)/$(BUILDDIR)/text @echo @echo "Build finished. The text files are in $(BUILDDIR)/text." -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man +man: staging + cd $(STAGINGDIR) && $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(PWD)/$(BUILDDIR)/man @echo @echo "Build finished. The manual pages are in $(BUILDDIR)/man." -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo +texinfo: staging + cd $(STAGINGDIR) && $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(PWD)/$(BUILDDIR)/texinfo @echo @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." @echo "Run \`make' in that directory to run these through makeinfo" \ "(use \`make info' here to do that automatically)." -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo +info: staging + cd $(STAGINGDIR) && $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(PWD)/$(BUILDDIR)/texinfo @echo "Running Texinfo files through makeinfo..." make -C $(BUILDDIR)/texinfo info @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale +gettext: staging + cd $(STAGINGDIR) && $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(PWD)/$(BUILDDIR)/locale @echo @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes +changes: staging + cd $(STAGINGDIR) && $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(PWD)/$(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck +linkcheck: staging + cd $(STAGINGDIR) && $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(PWD)/$(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest +doctest: staging + cd $(STAGINGDIR) && $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(PWD)/$(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." -coverage: - $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage +coverage: staging + cd $(STAGINGDIR) && $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(PWD)/$(BUILDDIR)/coverage @echo "Testing of coverage in the sources finished, look at the " \ "results in $(BUILDDIR)/coverage/python.txt." -xml: - $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml +xml: staging + cd $(STAGINGDIR) && $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(PWD)/$(BUILDDIR)/xml @echo @echo "Build finished. The XML files are in $(BUILDDIR)/xml." -pseudoxml: - $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml +pseudoxml: staging + cd $(STAGINGDIR) && $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(PWD)/$(BUILDDIR)/pseudoxml @echo @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/docs/conf.py b/docs/conf.py index 4a0455214db3..424eca236517 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,18 +30,30 @@ # All configuration values have a default; values that are commented out # serve to show the default. import gc -import sys +import importlib.util import inspect -import os, subprocess +import os +from pathlib import Path import shlex +import subprocess +import sys + import sphinx_gallery + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__))) -sys.path.insert(0, os.path.join(curr_path, "../python/")) -sys.path.insert(0, os.path.join(curr_path, "../vta/python")) +curr_path = Path(__file__).expanduser().absolute().parent +if curr_path.name == "_staging": + # Can't use curr_path.parent, because sphinx_gallery requires a relative path. + tvm_path = Path(os.pardir, os.pardir) +else: + tvm_path = Path(os.pardir) + + +sys.path.insert(0, str(tvm_path.joinpath("python"))) +sys.path.insert(0, str(tvm_path.joinpath("vta", "python"))) # -- General configuration ------------------------------------------------ @@ -56,7 +68,7 @@ def git_describe_version(original_version): """Get git describe version.""" - ver_py = os.path.join(curr_path, "..", "version.py") + ver_py = tvm_path.joinpath("version.py") libver = {"__file__": ver_py} exec(compile(open(ver_py, "rb").read(), ver_py, "exec"), libver, libver) _, gd_version = libver["git_describe_version"]() @@ -118,7 +130,7 @@ def git_describe_version(original_version): # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ["_build"] +exclude_patterns = ["_build", "_staging"] # The reST default role (used for this markup: `text`) to use for all # documents. @@ -197,24 +209,24 @@ def git_describe_version(original_version): from sphinx_gallery.sorting import ExplicitOrder -examples_dirs = ["../tutorials/", "../vta/tutorials/"] +examples_dirs = [tvm_path.joinpath("tutorials"), tvm_path.joinpath("vta", "tutorials")] gallery_dirs = ["tutorials", "vta/tutorials"] subsection_order = ExplicitOrder( [ - "../tutorials/get_started", - "../tutorials/frontend", - "../tutorials/language", - "../tutorials/optimize", - "../tutorials/autotvm", - "../tutorials/auto_scheduler", - "../tutorials/dev", - "../tutorials/topi", - "../tutorials/deployment", - "../tutorials/micro", - "../vta/tutorials/frontend", - "../vta/tutorials/optimize", - "../vta/tutorials/autotvm", + str(tvm_path.joinpath("tutorials", "get_started")), + str(tvm_path.joinpath("tutorials", "frontend")), + str(tvm_path.joinpath("tutorials", "language")), + str(tvm_path.joinpath("tutorials", "optimize")), + str(tvm_path.joinpath("tutorials", "autotvm")), + str(tvm_path.joinpath("tutorials", "auto_scheduler")), + str(tvm_path.joinpath("tutorials", "dev")), + str(tvm_path.joinpath("tutorials", "topi")), + str(tvm_path.joinpath("tutorials", "deployment")), + str(tvm_path.joinpath("tutorials", "micro")), + str(tvm_path.joinpath("vta", "tutorials", "frontend")), + str(tvm_path.joinpath("vta", "tutorials", "optimize")), + str(tvm_path.joinpath("vta", "tutorials", "autotvm")), ] ) From ba9b57bb36a82f9f9f96fe562a8394b7502d9624 Mon Sep 17 00:00:00 2001 From: Eric Lunderberg Date: Thu, 12 Aug 2021 11:26:37 -0500 Subject: [PATCH 2/4] Updates based on reviews. --- docs/Makefile | 10 ++++++++++ docs/conf.py | 33 +++++++++++++++++---------------- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/docs/Makefile b/docs/Makefile index 1922d056a130..e04e324a0f80 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -64,6 +64,10 @@ help: clean: rm -rf $(BUILDDIR) rm -rf $(STAGINGDIR) + + # TODO(Lunderberg): Remove these lines once the CI steps have + # propagated. + # Remove folders that have since been relocated into # $(STAGINGDIR). This allows `task_sphinx_precheck.sh` to # run, even if a commit that predates $(STAGINGDIR) was @@ -74,6 +78,12 @@ clean: rm -rf vta/tutorials staging: + # Prepare the staging directory. Sphinx gallery automatically + # writes new .rst files into the current directory. This can + # cause issues when switching branches. By sequestering the + # auto-generated files into the staging directory, they can be + # removed without knowing the exact directory. + mkdir -p $(STAGINGDIR) # Remove any symlinks that currently exist diff --git a/docs/conf.py b/docs/conf.py index 424eca236517..67e4f2c4098a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -52,8 +52,8 @@ tvm_path = Path(os.pardir) -sys.path.insert(0, str(tvm_path.joinpath("python"))) -sys.path.insert(0, str(tvm_path.joinpath("vta", "python"))) +sys.path.insert(0, str(tvm_path / "python")) +sys.path.insert(0, str(tvm_path / "vta" / "python")) # -- General configuration ------------------------------------------------ @@ -213,20 +213,21 @@ def git_describe_version(original_version): gallery_dirs = ["tutorials", "vta/tutorials"] subsection_order = ExplicitOrder( - [ - str(tvm_path.joinpath("tutorials", "get_started")), - str(tvm_path.joinpath("tutorials", "frontend")), - str(tvm_path.joinpath("tutorials", "language")), - str(tvm_path.joinpath("tutorials", "optimize")), - str(tvm_path.joinpath("tutorials", "autotvm")), - str(tvm_path.joinpath("tutorials", "auto_scheduler")), - str(tvm_path.joinpath("tutorials", "dev")), - str(tvm_path.joinpath("tutorials", "topi")), - str(tvm_path.joinpath("tutorials", "deployment")), - str(tvm_path.joinpath("tutorials", "micro")), - str(tvm_path.joinpath("vta", "tutorials", "frontend")), - str(tvm_path.joinpath("vta", "tutorials", "optimize")), - str(tvm_path.joinpath("vta", "tutorials", "autotvm")), + str(p) + for p in [ + tvm_path / "tutorials" / "get_started", + tvm_path / "tutorials" / "frontend", + tvm_path / "tutorials" / "language", + tvm_path / "tutorials" / "optimize", + tvm_path / "tutorials" / "autotvm", + tvm_path / "tutorials" / "auto_scheduler", + tvm_path / "tutorials" / "dev", + tvm_path / "tutorials" / "topi", + tvm_path / "tutorials" / "deployment", + tvm_path / "tutorials" / "micro", + tvm_path / "vta" / "tutorials" / "frontend", + tvm_path / "vta" / "tutorials" / "optimize", + tvm_path / "vta" / "tutorials" / "autotvm", ] ) From 0cd9c3d7314e7fc40b7c5b3f837daafa60fe5fb2 Mon Sep 17 00:00:00 2001 From: Eric Lunderberg Date: Thu, 12 Aug 2021 19:30:27 -0500 Subject: [PATCH 3/4] Changed graph_runtime references in deploy_classification.py to graph_executor --- vta/tutorials/frontend/deploy_classification.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vta/tutorials/frontend/deploy_classification.py b/vta/tutorials/frontend/deploy_classification.py index b2f909b9710a..139e30333f1e 100644 --- a/vta/tutorials/frontend/deploy_classification.py +++ b/vta/tutorials/frontend/deploy_classification.py @@ -52,7 +52,7 @@ import tvm from tvm import te from tvm import rpc, autotvm, relay -from tvm.contrib import graph_executor, utils, download, graph_runtime +from tvm.contrib import graph_executor, utils, download from tvm.contrib.debugger import debug_executor from tvm.relay import transform @@ -223,10 +223,10 @@ if env.TARGET == "intelfocl" or env.TARGET == "sim": ctxes = [remote.ext_dev(0), remote.cpu(0)] - m = graph_runtime.create(graph, lib, ctxes) + m = graph_executor.create(graph, lib, ctxes) else: # Graph runtime - m = graph_runtime.create(graph, lib, ctx) + m = graph_executor.create(graph, lib, ctx) ###################################################################### # Perform image classification inference From f5cd8299909f39354f3507ab29aa9891bf08e403 Mon Sep 17 00:00:00 2001 From: Eric Lunderberg Date: Fri, 13 Aug 2021 08:05:27 -0500 Subject: [PATCH 4/4] Removed unnecessary graph_runtime import from tune_alu_vta.py --- vta/tutorials/autotvm/tune_alu_vta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vta/tutorials/autotvm/tune_alu_vta.py b/vta/tutorials/autotvm/tune_alu_vta.py index 7b1fd411be57..3d38bbbbbd3d 100644 --- a/vta/tutorials/autotvm/tune_alu_vta.py +++ b/vta/tutorials/autotvm/tune_alu_vta.py @@ -28,7 +28,7 @@ import tvm from tvm import te from tvm import rpc, autotvm, relay -from tvm.contrib import graph_runtime, download +from tvm.contrib import download from tvm.autotvm.measure.measure_methods import request_remote from tvm.autotvm.tuner import XGBTuner, GATuner, RandomTuner, GridSearchTuner from tvm.autotvm import record