)")
+_re_parameter_group = re.compile(r"^> (.*)$", re.MULTILINE)
+_re_raises = re.compile(r"(.*)", re.DOTALL)
+_re_raisederrors = re.compile(r"(.*)", re.DOTALL)
+
+
+def get_signature_component(name, anchor, signature, object_doc, source_link=None, is_getset_desc=False):
+ """
+ Returns the svelte `Docstring` component string.
+
+ Args:
+ - **name** (`str`) -- The name of the function or class to document.
+ - **anchor** (`str`) -- The anchor name of the function or class that will be used for hash links.
+ - **signature** (`List(Dict(str,str))`) -- The signature of the object.
+ - **object_doc** (`str`) -- The docstring of the the object.
+ - **source_link** (Union[`str`, `None`], *optional*, defaults to `None`) -- The github source link of the the object.
+ - **is_getset_desc** (`bool`, *optional*, defaults to `False`) -- Whether the type of obj is `getset_descriptor`.
+ """
+
+ def inside_example_finder_closure(match, tag):
+ """
+ This closure find whether parameters and/or returns sections has example code block inside it
+ """
+ match_str = match.group(1)
+ examples_inside = _re_example_tags.search(match_str)
+ if examples_inside:
+ example_tag = examples_inside.group(1)
+ match_str = match_str.replace(example_tag, f"{example_tag}", 1)
+ return f"<{tag}>{match_str}"
+ return f"<{tag}>{match_str}"
+
+ def regex_closure(object_doc, regex):
+ """
+ This closure matches given regex & removes the matched group from object_doc
+ """
+ re_match = regex.search(object_doc)
+ object_doc = regex.sub("", object_doc)
+ match = None
+ if re_match:
+ _match = re_match.group(1).strip()
+ if len(_match):
+ match = _match
+ return object_doc, match
+
+ object_doc = _re_returns.sub(lambda m: inside_example_finder_closure(m, "returns"), object_doc)
+ object_doc = _re_parameters.sub(lambda m: inside_example_finder_closure(m, "parameters"), object_doc)
+
+ object_doc, parameters = regex_closure(object_doc, _re_parameters)
+ object_doc, return_description = regex_closure(object_doc, _re_returns)
+ object_doc, returntype = regex_closure(object_doc, _re_returntype)
+ object_doc, yield_description = regex_closure(object_doc, _re_yields)
+ object_doc, yieldtype = regex_closure(object_doc, _re_yieldtype)
+ object_doc, raise_description = regex_closure(object_doc, _re_raises)
+ object_doc, raisederrors = regex_closure(object_doc, _re_raisederrors)
+ object_doc = remove_example_tags(object_doc)
+ object_doc = hashlink_example_codeblock(object_doc, anchor)
+
+ svelte_str = "\n"
+
+ if source_link:
+ url = source_link
+ else:
+ url = ""
+
+ if len(name.split()) > 1:
+ obj_type = name.split()[0]
+ obj_name = name.split()[1]
+ else:
+ obj_type = ""
+ obj_name = name
+
+ title_str = f"
\n"
+ svelte_str += title_str
+
+ sig_list = []
+
+ for item in signature:
+ name = item['name']
+ value = item['val']
+
+ if value:
+ sig_list.append(f'{name}{value}')
+ else:
+ sig_list.append(name)
+
+ sig_str = ', '.join(sig_list)
+ # param_sig_str = param_sig_str.replace('`', '\`')
+ sig_str = f"<Signature>{{`{sig_str}`}}</Signature>\n"
+ svelte_str += r'{}'.format(sig_str)
+
+ # if is_getset_desc:
+ # svelte_str += "<isgetsetdescriptor>"
+
+ if parameters is not None:
+ parameters_str = ""
+ groups = _re_parameter_group.split(parameters)
+ group_default = groups.pop(0)
+ group_default = repr(group_default)
+ group_default = group_default.replace('`', '\`')
+ parameters_str += f"<Parameters>{{{group_default}}}</Parameters>\n"
+
+ # ignore for React + MDX
+ #
+ # n_groups = len(groups) // 2
+ # for idx in range(n_groups):
+ # id = idx + 1
+ # title, group = groups[2 * idx], groups[2 * idx + 1]
+ # parameters_str += f"<paramsdesc{id}title>{title}</paramsdesc{id}title>"
+ # parameters_str += f"<paramsdesc{id}>{group}</paramsdesc{id}>"
+
+ svelte_str += parameters_str
+ # svelte_str += f"<paramgroups>{n_groups}</paramgroups>"
+
+ if returntype is None:
+ returntype = ""
+ if return_description is None:
+ return_description = ""
+
+ if returntype or return_description:
+ svelte_str += f"<Returns name=\"{returntype}\" desc=\"{return_description}\"/>\n"
+
+ if yieldtype is None:
+ yieldtype = ""
+ if yield_description is None:
+ yield_description = ""
+ if yieldtype or yield_description:
+ svelte_str += f"<Yields name=\"{yieldtype}\" desc=\"{yield_description}\"/>\n"
+
+ if raise_description is None:
+ raise_description = ""
+ if raisederrors is None:
+ raisederrors = ""
+ if raisederrors or raise_description:
+ svelte_str += f"<Raises name=\"{raisederrors}\" desc=\"{raise_description}\"/>\n"
+
+ svelte_str += "</div>\n"
+
+ svelte_str += '<div>\n'
+ svelte_str += '<Divider name=\"Doc\" />'
+ svelte_str += f"\n{object_doc}\n"
+ svelte_str += '</div>\n'
+ return svelte_str
+
+
+# Re pattern to catch :obj:`xx`, :class:`xx`, :func:`xx` or :meth:`xx`.
+_re_rst_special_words = re.compile(r":(?:obj|func|class|meth):`([^`]+)`")
+# Re pattern to catch things between double backquotes.
+_re_double_backquotes = re.compile(r"(^|[^`])``([^`]+)``([^`]|$)")
+# Re pattern to catch example introduction.
+_re_rst_example = re.compile(r"^\s*Example.*::\s*$", flags=re.MULTILINE)
+
+
+def is_rst_docstring(docstring):
+ """
+ Returns `True` if `docstring` is written in rst.
+ """
+ if _re_rst_special_words.search(docstring) is not None:
+ return True
+ if _re_double_backquotes.search(docstring) is not None:
+ return True
+ if _re_rst_example.search(docstring) is not None:
+ return True
+ return False
+
+
+# Re pattern to catch example introduction & example code block.
+_re_example_codeblock = re.compile(r"((.*:\s+)?^```((?!```)(.|\n))*```)", re.MULTILINE)
+
+
+def hashlink_example_codeblock(object_doc, object_anchor):
+ """
+ Returns the svelte `ExampleCodeBlock` component string.
+
+ Args:
+ - **object_doc** (`str`) -- The docstring of the the object.
+ - **anchor** (`str`) -- The anchor name of the function or class that will be used for hash links.
+ """
+
+ example_id = 0
+
+ def add_example_svelte_blocks(match):
+ """
+ This closure matches `_re_example_codeblock` regex & creates `ExampleCodeBlock` svelte component
+ """
+ nonlocal example_id
+
+ example_id += 1
+ id_str = "" if example_id == 1 else f"-{example_id}"
+ example_anchor = f"{object_anchor}.example{id_str}"
+ content = match.group(1)
+
+ content = repr(content)
+ content = content.replace('`', '\`')
+ return f'<ExampleCode code={{{content}}} />\n'
+
+ object_doc = _re_example_codeblock.sub(add_example_svelte_blocks, object_doc)
+ return object_doc
+
+
+# Re pattern to numpystyle docstring (e.g Parameter -------).
+_re_numpydocstring = re.compile(r"(Parameter|Raise|Return|Yield)s?\n\s*----+\n")
+
+
+def is_numpy_docstring(docstring):
+ """
+ Returns `True` if `docstring` is written in numpystyle.
+ """
+ return _re_numpydocstring.search(docstring)
+
+
+def is_dataclass_autodoc(obj):
+ """
+ Returns boolean whether object's doc was generated automatically by `dataclass`.
+ """
+ if is_getset_descriptor(obj):
+ return False
+ try:
+ signature = str(inspect.signature(obj))
+ except ValueError:
+ # object doesn't have signature
+ return False
+ doc = obj.__doc__
+ doc_generated = obj.__name__ + signature
+ is_generated = doc in doc_generated
+ return is_generated
+
+
+def is_getset_descriptor(obj):
+ """
+ Returns boolean whether object is `getset_descriptor`.
+ """
+ # used by tokenizers @property bindings
+ obj_repr = str(type(obj))
+ return "getset_descriptor" in obj_repr
+
+
+def get_source_link(obj, page_info, version_tag_suffix="src/"):
+ """
+ Returns the link to the source code of an object on GitHub.
+ """
+ # Repo name defaults to package_name, but if provided in page_info, it will be used instead.
+ repo_name = page_info.get("repo_name", page_info.get("package_name"))
+ version_tag = page_info.get("version_tag", "main")
+ repo_owner = page_info.get("repo_owner", "huggingface")
+ base_link = f"https://github.com/{repo_owner}/{repo_name}/blob/{version_tag}/{version_tag_suffix}"
+ module = obj.__module__.replace(".", "/")
+ line_number = inspect.getsourcelines(obj)[1]
+ source_file = inspect.getsourcefile(obj)
+ if source_file.endswith("__init__.py"):
+ return f"{base_link}{module}/__init__.py#L{line_number}"
+ return f"{base_link}{module}.py#L{line_number}"
+
+
+def get_source_path(object_name, package):
+ """
+ Find a path to file in which given object was defined.
+
+ Args:
+ - object_name (`str`): The name of the object to retrieve.
+ - package (`types.ModuleType`): The package to look into.
+ """
+ obj = obj = find_object_in_package(object_name=object_name, package=package)
+ obj_path = inspect.getfile(obj)
+ return obj_path
+
+
+def document_object(object_name, package, page_info, full_name=True, anchor_name=None, version_tag_suffix="src/"):
+ """
+ Writes the document of a function, class or method.
+
+ Args:
+ object_name (`str`): The name of the object to document.
+ package (`types.ModuleType`): The package of the object.
+ full_name (`bool`, *optional*, defaults to `True`): Whether to write the full name of the object or not.
+ anchor_name (`str`, *optional*): The name to give to the anchor for this object.
+ version_tag_suffix (`str`, *optional*, defaults to `"src/"`):
+ Suffix to add after the version tag (e.g. 1.3.0 or main) in the documentation links.
+ For example, the default `"src/"` suffix will result in a base link as `https://github.com/{repo_owner}/{package_name}/blob/{version_tag}/src/`.
+ For example, `version_tag_suffix=""` will result in a base link as `https://github.com/{repo_owner}/{package_name}/blob/{version_tag}/`.
+ """
+ if page_info is None:
+ page_info = {}
+ if "package_name" not in page_info:
+ page_info["package_name"] = package.__name__
+ obj = find_object_in_package(object_name=object_name, package=package)
+ if obj is None:
+ raise ValueError(
+ f"Unable to find {object_name} in {package.__name__}. Make sure the path to that object is correct."
+ )
+ if isinstance(obj, property):
+ # Propreties have no __module__ or __name__ attributes, but their getter function does.
+ obj = obj.fget
+
+ if anchor_name is None:
+ anchor_name = get_shortest_path(obj, package)
+ if full_name and anchor_name is not None:
+ name = anchor_name
+ else:
+ name = obj.__name__
+
+ prefix = "class " if isinstance(obj, type) else ""
+ object_doc = ""
+ signature_name = prefix + name
+ signature = format_signature(obj)
+ check = None
+ if getattr(obj, "__doc__", None) is not None and len(obj.__doc__) > 0:
+ object_doc = obj.__doc__
+ if is_dataclass_autodoc(obj):
+ object_doc = ""
+ elif is_rst_docstring(object_doc):
+ object_doc = convert_rst_docstring_to_mdx(obj.__doc__, page_info)
+ else:
+ check = quality_check_docstring(object_doc, object_name=object_name)
+ object_doc = convert_md_docstring_to_mdx(obj.__doc__, page_info)
+
+ try:
+ source_link = get_source_link(obj, page_info, version_tag_suffix)
+ except (AttributeError, OSError, TypeError):
+ # tokenizers obj do NOT have `__module__` attribute & can NOT be used with inspect.getsourcelines
+ source_link = None
+ is_getset_desc = is_getset_descriptor(obj)
+ component = get_signature_component(
+ signature_name, anchor_name, signature, object_doc, source_link, is_getset_desc
+ )
+ documentation = "\n" + component + "\n"
+ return documentation, check
+
+
+def find_documented_methods(clas):
+ """
+ Find all the public methods of a given class that have a nonempty documentation, filtering the methods documented
+ the exact same way in a superclass.
+ """
+ public_attrs = {a: getattr(clas, a) for a in dir(clas) if not a.startswith("_")}
+ public_methods = {a: m for a, m in public_attrs.items() if callable(m) and not isinstance(m, type)}
+ documented_methods = {
+ a: m for a, m in public_methods.items() if getattr(m, "__doc__", None) is not None and len(m.__doc__) > 0
+ }
+
+ superclasses = clas.mro()[1:]
+ for superclass in superclasses:
+ superclass_methods = {a: getattr(superclass, a) for a in documented_methods.keys() if hasattr(superclass, a)}
+ documented_methods = {
+ a: m
+ for a, m in documented_methods.items()
+ if (
+ a not in superclass_methods
+ or getattr(superclass_methods[a], "__doc__", None) is None
+ or m.__doc__ != superclass_methods[a].__doc__
+ )
+ }
+ return list(documented_methods.keys())
+
+
+def autodoc(object_name, package, methods=None, return_anchors=False, page_info=None, version_tag_suffix="src/"):
+ """
+ Generates the documentation of an object, with a potential filtering on the methods for a class.
+
+ Args:
+ object_name (`str`): The name of the function or class to document.
+ package (`types.ModuleType`): The package of the object.
+ methods (`List[str]`, *optional*):
+ A list of methods to document if `obj` is a class. If nothing is passed, all public methods with a new
+ docstring compared to the superclasses are documented. If a list of methods is passed and ou want to add
+ all those methods, the key "all" will add them.
+ return_anchors (`bool`, *optional*, defaults to `False`):
+ Whether or not to return the list of anchors generated.
+ page_info (`Dict[str, str]`, *optional*): Some information about the page.
+ version_tag_suffix (`str`, *optional*, defaults to `"src/"`):
+ Suffix to add after the version tag (e.g. 1.3.0 or main) in the documentation links.
+ For example, the default `"src/"` suffix will result in a base link as `https://github.com/{repo_owner}/{package_name}/blob/{version_tag}/src/`.
+ For example, `version_tag_suffix=""` will result in a base link as `https://github.com/{repo_owner}/{package_name}/blob/{version_tag}/`.
+ """
+ if page_info is None:
+ page_info = {}
+ if "package_name" not in page_info:
+ page_info["package_name"] = package.__name__
+
+ errors = []
+ obj = find_object_in_package(object_name=object_name, package=package)
+ documentation, check = document_object(
+ object_name=object_name, package=package, page_info=page_info, version_tag_suffix=version_tag_suffix
+ )
+ if check is not None:
+ errors.append(check)
+
+ if return_anchors:
+ anchors = [get_shortest_path(obj, package)]
+ else:
+ anchors = None
+
+ if isinstance(obj, type):
+ documentation, check = document_object(
+ object_name=object_name, package=package, page_info=page_info, version_tag_suffix=version_tag_suffix
+ )
+ if check is not None:
+ errors.append(check)
+ if methods is None:
+ methods = find_documented_methods(obj)
+ elif "all" in methods:
+ methods.remove("all")
+ methods_to_add = find_documented_methods(obj)
+ methods.extend([m for m in methods_to_add if m not in methods])
+ for method in methods:
+ if anchors:
+ anchor_name = f"{anchors[0]}.{method}"
+ else:
+ anchor_name = method
+
+ method_doc, check = document_object(
+ object_name=f"{object_name}.{method}",
+ package=package,
+ page_info=page_info,
+ full_name=False,
+ anchor_name=anchor_name,
+ version_tag_suffix=version_tag_suffix,
+ )
+ if check is not None:
+ errors.append(check)
+ documentation += f'\n<div">' + method_doc + "</div>"
+ if return_anchors:
+ # The anchor name of the method might be different from its
+ method = find_object_in_package(f"{anchors[0]}.{method}", package=package)
+ method_name = get_shortest_path(method, package=package)
+ if anchor_name == method_name or method_name is None:
+ anchors.append(anchor_name)
+ else:
+ anchors.append((anchor_name, method_name))
+ documentation = f'<DocStringContainer>\n' + documentation + "</DocStringContainer>\n"
+
+ return (documentation, anchors, errors) if return_anchors else documentation
+
+
+def resolve_links_in_text(text, package, mapping, page_info):
+ """
+ Resolve links of the form [`SomeClass`] to the link in the documentation to `SomeClass`.
+
+ Args:
+ text (`str`): The text in which to convert the links.
+ package (`types.ModuleType`): The package in which to search objects for.
+ mapping (`Dict[str, str]`): The map from anchor names of objects to their page in the documentation.
+ page_info (`Dict[str, str]`): Some information about the page.
+ """
+ package_name = page_info.get("package_name", package.__name__)
+ version = page_info.get("version", "main")
+ language = page_info.get("language", "en")
+
+ prefix = f"/docs/{package_name}/{version}/{language}/"
+
+ def _resolve_link(search):
+ object_or_param_name, last_char = search.groups()
+ # Deal with external libs first.
+ lib_name = object_or_param_name.split(".")[0]
+ if lib_name.startswith("~"):
+ lib_name = lib_name[1:]
+ if lib_name in HUGGINFACE_LIBS and lib_name != package_name:
+ link = get_external_object_link(object_or_param_name, page_info)
+ return f"{link}{last_char}"
+ object_name, param_name = None, None
+ # If `#` is in the name, assume it's a link to the function/method parameter.
+ if "#" in object_or_param_name:
+ object_name_for_param = object_or_param_name.split("#", 1)[0]
+ # Strip preceding `~` if it's there.
+ object_name_for_param = (
+ object_name_for_param[1:] if object_name_for_param.startswith("~") else object_name_for_param
+ )
+ obj = find_object_in_package(object_name_for_param, package)
+ param_name = object_or_param_name.split("#", 1)[-1]
+ # If the name begins with `~`, we shortcut to the last part.
+ elif object_or_param_name.startswith("~"):
+ obj = find_object_in_package(object_or_param_name[1:], package)
+ object_name = object_or_param_name.split(".")[-1]
+ else:
+ obj = find_object_in_package(object_or_param_name, package)
+ object_name = object_or_param_name
+ # Object not found, return the original link text.
+ if obj is None:
+ return f"`{object_or_param_name}`{last_char}"
+
+ link_name = object_name if param_name is None else param_name
+
+ # If the link points to an object and the object is not a class, we add ()
+ if param_name is None and not isinstance(obj, (type, property)):
+ link_name = f"{link_name}()"
+
+ # Link to the anchor
+ anchor = get_shortest_path(obj, package)
+ if anchor not in mapping:
+ return f"`{link_name}`{last_char}"
+ page = f"{prefix}{mapping[anchor]}"
+ if param_name:
+ anchor = f"{anchor}.{param_name}"
+ if "#" in page:
+ return f"[{link_name}]({page}){last_char}"
+ else:
+ return f"[{link_name}]({page}#{anchor}){last_char}"
+
+ return re.sub(r"\[`([^`]+)`\]([^\(])", _resolve_link, text)
+
+
+# Re pattern that catches the start of a block code with potential indent.
+_re_start_code_block = re.compile(r"^\s*```.*$", flags=re.MULTILINE)
+# Re pattern that catches return blocks of the form `Return:`.
+_re_returns_block = re.compile("^\s*Returns?:\s*$")
+
+
+def quality_check_docstring(docstring, object_name=None):
+ """
+ Check if a docstring is not going to generate a common error on moon-landing, by asserting it does not have:
+
+ - an empty Return block
+ - two (or more) Return blocks
+ - a code sample not properly closed.
+
+ This function only returns an error message and does not raise an exception, as we will raise one single exception
+ with all the problems at the end.
+
+ Args:
+ docstring (`str`): The docstring to check.
+ obejct_name (`str`, *optional*): The name of the object being documented.
+ Will be added to the error message if passed.
+
+ Returns:
+ Optional `str`: Returns `None` if the docstring is correct and an error message otherwise.
+ """
+
+ lines = docstring.split("\n")
+ in_code = False
+ code_indent = 0
+ return_blocks = 0
+ error_message = ""
+
+ for idx, line in enumerate(lines):
+ if not in_code and _re_start_code_block.search(line) is not None:
+ in_code = True
+ code_indent = find_indent(line)
+ elif in_code and line.rstrip() == " " * code_indent + "```":
+ in_code = False
+ elif _re_returns_block.search(line) is not None:
+ next_line_idx = idx + 1
+ while next_line_idx < len(lines) and is_empty_line(lines[next_line_idx]):
+ next_line_idx += 1
+ if next_line_idx >= len(lines) or find_indent(lines[next_line_idx]) <= find_indent(line):
+ error_message += "- The return block is empty.\n"
+ else:
+ return_blocks += 1
+
+ if in_code:
+ error_message += "- A code block has been opened but is not closed.\n"
+ if return_blocks >= 2:
+ error_message += f"- There are {return_blocks} Returns block. Only one max is supported.\n"
+
+ if len(error_message) == 0:
+ return
+
+ if object_name is not None:
+ error_message = (
+ f"The docstring of {object_name} comports the following issue(s) and needs fixing:\n" + error_message
+ )
+
+ return error_message
+ return error_message
+ return error_message
+ return error_message
diff --git a/doc-build/third_party/hf-doc-builder/src/doc_builder/build_doc.py b/doc-build/third_party/hf-doc-builder/src/doc_builder/build_doc.py
new file mode 100644
index 00000000..ab50feb3
--- /dev/null
+++ b/doc-build/third_party/hf-doc-builder/src/doc_builder/build_doc.py
@@ -0,0 +1,570 @@
+# coding=utf-8
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import importlib
+import os
+import re
+import shutil
+import zlib
+from pathlib import Path
+
+import yaml
+from tqdm import tqdm
+
+from .autodoc import autodoc, find_object_in_package, get_source_path, resolve_links_in_text
+from .convert_md_to_mdx import convert_md_to_mdx
+from .convert_rst_to_mdx import convert_rst_to_mdx, find_indent, is_empty_line
+from .convert_to_notebook import generate_notebooks_from_file
+from .frontmatter_node import FrontmatterNode
+from .utils import get_doc_config, read_doc_config
+
+
+_re_autodoc = re.compile("^\s*\[\[autodoc\]\]\s+(\S+)\s*$")
+_re_list_item = re.compile("^\s*-\s+(\S+)\s*$")
+
+
+def resolve_open_in_colab(content, page_info):
+ """
+ Replaces [[open-in-colab]] special markers by the proper svelte component.
+
+ Args:
+ content (`str`): The documentation to treat.
+ page_info (`Dict[str, str]`, *optional*): Some information about the page.
+ """
+ if "[[open-in-colab]]" not in content:
+ return content
+
+ package_name = page_info["package_name"]
+ language = page_info.get("language", "en")
+ page_name = Path(page_info["page"]).stem
+ nb_prefix = f"/github/huggingface/notebooks/blob/main/{package_name}_doc/{language}/"
+ nb_prefix_colab = f"https://colab.research.google.com{nb_prefix}"
+ nb_prefix_awsstudio = f"https://studiolab.sagemaker.aws/import{nb_prefix}"
+ links = [
+ ("Mixed", f"{nb_prefix_colab}{page_name}.ipynb"),
+ ("PyTorch", f"{nb_prefix_colab}pytorch/{page_name}.ipynb"),
+ ("TensorFlow", f"{nb_prefix_colab}tensorflow/{page_name}.ipynb"),
+ ("Mixed", f"{nb_prefix_awsstudio}{page_name}.ipynb"),
+ ("PyTorch", f"{nb_prefix_awsstudio}pytorch/{page_name}.ipynb"),
+ ("TensorFlow", f"{nb_prefix_awsstudio}tensorflow/{page_name}.ipynb"),
+ ]
+ formatted_links = [' {label: "' + key + '", value: "' + value + '"},' for key, value in links]
+
+ svelte_component = """<DocNotebookDropdown
+ classNames="absolute z-10 right-0 top-0"
+ options={[
+"""
+ svelte_component += "\n".join(formatted_links)
+ svelte_component += "\n]} />"
+
+ return content.replace("[[open-in-colab]]", svelte_component)
+
+
+def resolve_autodoc(content, package, return_anchors=False, page_info=None, version_tag_suffix="src/"):
+ """
+ Replaces [[autodoc]] special syntax by the corresponding generated documentation in some content.
+
+ Args:
+ content (`str`): The documentation to treat.
+ package (`types.ModuleType`): The package where to look for objects to document.
+ return_anchors (`bool`, *optional*, defaults to `False`):
+ Whether or not to return the list of anchors generated.
+ page_info (`Dict[str, str]`, *optional*): Some information about the page.
+ version_tag_suffix (`str`, *optional*, defaults to `"src/"`):
+ Suffix to add after the version tag (e.g. 1.3.0 or main) in the documentation links.
+ For example, the default `"src/"` suffix will result in a base link as `https://github.com/huggingface/{package_name}/blob/{version_tag}/src/`.
+ For example, `version_tag_suffix=""` will result in a base link as `https://github.com/huggingface/{package_name}/blob/{version_tag}/`.
+ """
+ idx_last_heading = None
+ is_inside_codeblock = False
+ lines = content.split("\n")
+ new_lines = []
+ source_files = None
+ if return_anchors:
+ anchors = []
+ errors = []
+ idx = 0
+ while idx < len(lines):
+ if _re_autodoc.search(lines[idx]) is not None:
+ object_name = _re_autodoc.search(lines[idx]).groups()[0]
+ autodoc_indent = find_indent(lines[idx])
+ idx += 1
+ while idx < len(lines) and is_empty_line(lines[idx]):
+ idx += 1
+ if (
+ idx < len(lines)
+ and find_indent(lines[idx]) > autodoc_indent
+ and _re_list_item.search(lines[idx]) is not None
+ ):
+ methods = []
+ methods_indent = find_indent(lines[idx])
+ while is_empty_line(lines[idx]) or (
+ find_indent(lines[idx]) == methods_indent and _re_list_item.search(lines[idx]) is not None
+ ):
+ if not is_empty_line(lines[idx]):
+ methods.append(_re_list_item.search(lines[idx]).groups()[0])
+ idx += 1
+ if idx >= len(lines):
+ break
+ else:
+ methods = None
+ doc = autodoc(
+ object_name,
+ package,
+ methods=methods,
+ return_anchors=return_anchors,
+ page_info=page_info,
+ version_tag_suffix=version_tag_suffix,
+ )
+ if return_anchors:
+ if len(doc[1]) and idx_last_heading is not None:
+ object_anchor = doc[1][0]
+ new_lines[idx_last_heading] += f"[[{object_anchor}]]"
+ idx_last_heading = None
+ anchors.extend(doc[1])
+ errors.extend(doc[2])
+ doc = doc[0]
+ new_lines.append(doc)
+
+ try:
+ source_files = source_files = get_source_path(object_name, package)
+ except (AttributeError, OSError, TypeError):
+ # tokenizers obj do NOT have `__module__` attribute & can NOT be used with inspect.getfile
+ source_files = None
+ else:
+ new_lines.append(lines[idx])
+ if lines[idx].startswith("```"):
+ is_inside_codeblock = not is_inside_codeblock
+ if lines[idx].startswith("#") and not is_inside_codeblock:
+ idx_last_heading = len(new_lines) - 1
+ idx += 1
+
+ new_content = "\n".join(new_lines)
+
+ return (new_content, anchors, source_files, errors) if return_anchors else new_content
+
+
+def build_mdx_files(package, doc_folder, output_dir, page_info, version_tag_suffix):
+ """
+ Build the MDX files for a given package.
+
+ Args:
+ package (`types.ModuleType`): The package where to look for objects to document.
+ doc_folder (`str` or `os.PathLike`): The folder where the doc source files are.
+ output_dir (`str` or `os.PathLike`): The folder where to put the files built.
+ page_info (`Dict[str, str]`): Some information about the page.
+ version_tag_suffix (`str`, *optional*, defaults to `"src/"`):
+ Suffix to add after the version tag (e.g. 1.3.0 or main) in the documentation links.
+ For example, the default `"src/"` suffix will result in a base link as `https://github.com/huggingface/{package_name}/blob/{version_tag}/src/`.
+ For example, `version_tag_suffix=""` will result in a base link as `https://github.com/huggingface/{package_name}/blob/{version_tag}/`.
+ """
+ doc_folder = Path(doc_folder)
+ output_dir = Path(output_dir)
+ os.makedirs(output_dir, exist_ok=True)
+ anchor_mapping = {}
+ source_files_mapping = {}
+
+ if "package_name" not in page_info:
+ page_info["package_name"] = package.__name__
+
+ all_files = list(doc_folder.glob("**/*"))
+ all_errors = []
+ for file in tqdm(all_files, desc="Building the MDX files"):
+ new_anchors = None
+ errors = None
+ page_info["path"] = file
+ try:
+ if file.suffix in [".md", ".mdx"]:
+ dest_file = output_dir / (file.with_suffix(".mdx").relative_to(doc_folder))
+ page_info["page"] = file.with_suffix(".html").relative_to(doc_folder).as_posix()
+ os.makedirs(dest_file.parent, exist_ok=True)
+ with open(file, "r", encoding="utf-8-sig") as reader:
+ content = reader.read()
+ content = convert_md_to_mdx(content, page_info)
+ content = resolve_open_in_colab(content, page_info)
+ content, new_anchors, source_files, errors = resolve_autodoc(
+ content, package, return_anchors=True, page_info=page_info, version_tag_suffix=version_tag_suffix
+ )
+ if source_files is not None:
+ source_files_mapping[source_files] = str(file)
+ with open(dest_file, "w", encoding="utf-8") as writer:
+ writer.write(content)
+ # Make sure we clean up for next page.
+ del page_info["page"]
+ elif file.suffix in [".rst"]:
+ dest_file = output_dir / (file.with_suffix(".mdx").relative_to(doc_folder))
+ page_info["page"] = file.with_suffix(".html").relative_to(doc_folder)
+ os.makedirs(dest_file.parent, exist_ok=True)
+ with open(file, "r", encoding="utf-8") as reader:
+ content = reader.read()
+ content = convert_rst_to_mdx(content, page_info)
+ content = resolve_open_in_colab(content, page_info)
+ content, new_anchors, source_files, errors = resolve_autodoc(
+ content, package, return_anchors=True, page_info=page_info, version_tag_suffix=version_tag_suffix
+ )
+ if source_files is not None:
+ source_files_mapping[source_files] = str(file)
+ with open(dest_file, "w", encoding="utf-8") as writer:
+ writer.write(content)
+ # Make sure we clean up for next page.
+ del page_info["page"]
+ elif file.is_file() and "__" not in str(file):
+ # __ is a reserved svelte file/folder prefix
+ dest_file = output_dir / (file.relative_to(doc_folder))
+ os.makedirs(dest_file.parent, exist_ok=True)
+ shutil.copy(file, dest_file)
+
+ except Exception as e:
+ raise type(e)(f"There was an error when converting {file} to the MDX format.\n" + e.args[0]) from e
+
+ if new_anchors is not None:
+ page_name = str(file.with_suffix("").relative_to(doc_folder))
+ for anchor in new_anchors:
+ if isinstance(anchor, tuple):
+ anchor_mapping.update({a: f"{page_name}#{anchor[0]}" for a in anchor[1:]})
+ anchor = anchor[0]
+ anchor_mapping[anchor] = page_name
+
+ if errors is not None:
+ all_errors.extend(errors)
+
+ if len(all_errors) > 0:
+ raise ValueError(
+ "The deployment of the documentation will fail because of the following errors:\n" + "\n".join(all_errors)
+ )
+
+ return anchor_mapping, source_files_mapping
+
+
+def resolve_links(doc_folder, package, mapping, page_info):
+ """
+ Resolve links of the form [`SomeClass`] to the link in the documentation to `SomeClass` for all files in a
+ folder.
+
+ Args:
+ doc_folder (`str` or `os.PathLike`): The folder in which to look for files.
+ package (`types.ModuleType`): The package in which to search objects for.
+ mapping (`Dict[str, str]`): The map from anchor names of objects to their page in the documentation.
+ page_info (`Dict[str, str]`): Some information about the page.
+ """
+ doc_folder = Path(doc_folder)
+ all_files = list(doc_folder.glob("**/*.mdx"))
+ for file in tqdm(all_files, desc="Resolving internal links"):
+ with open(file, "r", encoding="utf-8") as reader:
+ content = reader.read()
+ content = resolve_links_in_text(content, package, mapping, page_info)
+ with open(file, "w", encoding="utf-8") as writer:
+ writer.write(content)
+
+
+def generate_frontmatter_in_text(text, file_name=None):
+ """
+ Adds frontmatter & turns markdown headers into markdown headers with hash links.
+
+ Args:
+ text (`str`): The text in which to convert the links.
+ """
+ is_disabled = "<!-- DISABLE-FRONTMATTER-SECTIONS -->" in text
+ text = text.split("\n")
+ root = None
+ is_inside_codeblock = False
+ for idx, line in enumerate(text):
+ if line.startswith("```"):
+ is_inside_codeblock = not is_inside_codeblock
+ if is_inside_codeblock or is_empty_line(line):
+ continue
+ header_search = re.search(r"^(#+)\s+(\S.*)$", line)
+ if header_search is None:
+ continue
+ first_word, title = header_search.groups()
+ header_level = len(first_word)
+ serach_local = re.search(r"\[\[(.*)]]", title)
+ if serach_local:
+ # id/local already exists
+ local = serach_local.group(1)
+ title = re.sub(r"\[\[(.*)]]", "", title)
+ else:
+ # create id/local
+ local = re.sub(r"[^a-z0-9\s]+", "", title.lower())
+ local = re.sub(r"\s{2,}", " ", local.strip()).replace(" ", "-")
+ text[idx] = f'<h{header_level} id="{local}">{title}</h{header_level}>'
+ node = FrontmatterNode(title, local)
+ if header_level == 1:
+ root = node
+ # doc writers may choose to disable frontmatter generation
+ # currently used in Quiz sections of hf course
+ if is_disabled:
+ break
+ else:
+ if root is None:
+ raise ValueError(
+ f"{file_name} does not contain a level 1 header (more commonly known as title) before the first "
+ " second (or more) level header. Make sure to include one!"
+ )
+ root.add_child(node, header_level)
+
+ frontmatter = root.get_frontmatter()
+ text = "\n".join(text)
+ text = frontmatter + text
+ return text
+
+
+def generate_frontmatter(doc_folder):
+ """
+ Adds frontmatter & turns markdown headers into markdown headers with hash links for all files in a folder.
+
+ Args:
+ doc_folder (`str` or `os.PathLike`): The folder in which to look for files.
+ """
+ doc_folder = Path(doc_folder)
+ all_files = list(doc_folder.glob("**/*.mdx"))
+ for file_name in tqdm(all_files, desc="Generating frontmatter"):
+ # utf-8-sig is needed to correctly open community.md file
+ with open(file_name, "r", encoding="utf-8-sig") as reader:
+ content = reader.read()
+ content = generate_frontmatter_in_text(content, file_name=file_name)
+ with open(file_name, "w", encoding="utf-8") as writer:
+ writer.write(content)
+
+
+def build_notebooks(doc_folder, notebook_dir, package=None, mapping=None, page_info=None):
+ """
+ Build the notebooks associated to the MDX files in the documentation with an [[open-in-colab]] marker.
+
+ Args:
+ doc_folder (`str` or `os.PathLike`): The folder where the doc source files are.
+ notebook_dir_dir (`str` or `os.PathLike`): Where to save the generated notebooks
+ package (`types.ModuleType`, *optional*):
+ The package in which to search objects for (needs to be passed to resolve doc links).
+ mapping (`Dict[str, str]`, *optional*):
+ The map from anchor names of objects to their page in the documentation (needs to be passed to resolve doc
+ links).
+ page_info (`Dict[str, str]`, *optional*):
+ Some information about the page (needs to be passed to resolve doc links).
+ """
+ doc_folder = Path(doc_folder)
+
+ if "package_name" not in page_info:
+ page_info["package_name"] = package.__name__
+
+ mdx_files = list(doc_folder.glob("**/*.mdx"))
+ for file in tqdm(mdx_files, desc="Building the notebooks"):
+ with open(file, "r", encoding="utf-8") as f:
+ if "[[open-in-colab]]" not in f.read():
+ continue
+ try:
+ page_info["page"] = file.with_suffix(".html").relative_to(doc_folder)
+ generate_notebooks_from_file(file, notebook_dir, package=package, mapping=mapping, page_info=page_info)
+ # Make sure we clean up for next page.
+ del page_info["page"]
+
+ except Exception as e:
+ raise type(e)(f"There was an error when converting {file} to a notebook.\n" + e.args[0]) from e
+
+
+def build_doc(
+ package_name,
+ doc_folder,
+ output_dir,
+ clean=True,
+ version="main",
+ version_tag="main",
+ language="en",
+ notebook_dir=None,
+ is_python_module=False,
+ watch_mode=False,
+ version_tag_suffix="src/",
+ repo_owner="huggingface",
+ repo_name=None,
+):
+ """
+ Build the documentation of a package.
+
+ Args:
+ package_name (`str`): The name of the package.
+ doc_folder (`str` or `os.PathLike`): The folder in which the source documentation of the package is.
+ output_dir (`str` or `os.PathLike`):
+ The folder in which to put the built documentation. Will be created if it does not exist.
+ clean (`bool`, *optional*, defaults to `True`):
+ Whether or not to delete the content of the `output_dir` if that directory exists.
+ version (`str`, *optional*, defaults to `"main"`): The name of the version of the doc.
+ version_tag (`str`, *optional*, defaults to `"main"`): The name of the version tag (on GitHub) of the doc.
+ language (`str`, *optional*, defaults to `"en"`): The language of the doc.
+ notebook_dir (`str` or `os.PathLike`, *optional*):
+ If provided, where to save the notebooks generated from the doc file with an [[open-in-colab]] marker.
+ is_python_module (`bool`, *optional*, defaults to `False`):
+ Whether the docs being built are for python module. (For example, HF Course is not a python module).
+ watch_mode (`bool`, *optional*, default to `False`):
+ If `True`, disables the toc tree check and sphinx objects.inv builds since they are not needed
+ when this mode is active.
+ version_tag_suffix (`str`, *optional*, defaults to `"src/"`):
+ Suffix to add after the version tag (e.g. 1.3.0 or main) in the documentation links.
+ For example, the default `"src/"` suffix will result in a base link as `https://github.com/huggingface/{package_name}/blob/{version_tag}/src/`.
+ For example, `version_tag_suffix=""` will result in a base link as `https://github.com/huggingface/{package_name}/blob/{version_tag}/`.
+ repo_owner (`str`, *optional*, defaults to `"huggingface"`):
+ The owner of the repository on GitHub. In most cases, this is `"huggingface"`. However, for the `timm` library, the owner is `"rwightman"`.
+ repo_name (`str`, *optional*, defaults to `package_name`):
+ The name of the repository on GitHub. In most cases, this is the same as `package_name`. However, for the `timm` library, the name is `"pytorch-image-models"` instead of `"timm"`.
+ """
+ page_info = {
+ "version": version,
+ "version_tag": version_tag,
+ "language": language,
+ "package_name": package_name,
+ "repo_owner": repo_owner,
+ "repo_name": repo_name if repo_name is not None else package_name,
+ }
+ if clean and Path(output_dir).exists():
+ shutil.rmtree(output_dir)
+
+ read_doc_config(doc_folder)
+
+ package = importlib.import_module(package_name) if is_python_module else None
+ anchors_mapping, source_files_mapping = build_mdx_files(
+ package, doc_folder, output_dir, page_info, version_tag_suffix=version_tag_suffix
+ )
+ if not watch_mode:
+ sphinx_refs = check_toc_integrity(doc_folder, output_dir)
+ sphinx_refs.extend(convert_anchors_mapping_to_sphinx_format(anchors_mapping, package))
+ if is_python_module:
+ if not watch_mode:
+ build_sphinx_objects_ref(sphinx_refs, output_dir, page_info)
+ resolve_links(output_dir, package, anchors_mapping, page_info)
+ generate_frontmatter(output_dir)
+
+ if notebook_dir is not None:
+ if clean and Path(notebook_dir).exists():
+ for nb_file in Path(notebook_dir).glob("**/*.ipynb"):
+ os.remove(nb_file)
+ build_notebooks(doc_folder, notebook_dir, package=package, mapping=anchors_mapping, page_info=page_info)
+
+ return source_files_mapping
+
+
+def check_toc_integrity(doc_folder, output_dir):
+ """
+ Checks all the MDX files obtained after building the documentation are present in the table of contents.
+
+ Args:
+ doc_folder (`str` or `os.PathLike`): The folder where the source files of the documentation lie.
+ output_dir (`str` or `os.PathLike`): The folder where the doc is built.
+ """
+ output_dir = Path(output_dir)
+ doc_files = [str(f.relative_to(output_dir).with_suffix("").as_posix()) for f in output_dir.glob("**/*.mdx")]
+
+ toc_file = Path(doc_folder) / "_toctree.yml"
+ with open(toc_file, "r", encoding="utf-8") as f:
+ toc = yaml.safe_load(f.read())
+
+ toc_sections = []
+ sphinx_refs = []
+ # We don't just loop directly in toc as we will add more into it as we un-nest things.
+ while len(toc) > 0:
+ part = toc.pop(0)
+ if "local" in part:
+ toc_sections.append(part["local"])
+ if "sections" not in part:
+ continue
+ toc_sections.extend([sec["local"] for sec in part["sections"] if "local" in sec])
+ for sec in part["sections"]:
+ if "local_fw" in sec:
+ toc_sections.extend(sec["local_fw"].values())
+ # There should be one sphinx ref per page
+ for sec in part["sections"]:
+ if "local" in sec:
+ sphinx_refs.append(f"{sec['local']} std:doc -1 {sec['local']} {sec['title']}")
+ # Toc has some nested sections in the API doc for instance, so we recurse.
+ toc.extend([sec for sec in part["sections"] if "sections" in sec])
+
+ files_not_in_toc = [f for f in doc_files if f not in toc_sections]
+ doc_config = get_doc_config()
+ disable_toc_check = getattr(doc_config, "disable_toc_check", False)
+ if len(files_not_in_toc) > 0 and not disable_toc_check:
+ message = "\n".join([f"- {f}" for f in files_not_in_toc])
+ raise RuntimeError(
+ "The following files are not present in the table of contents:\n" + message + f"\nAdd them to {toc_file}."
+ )
+
+ files_not_exist = [f for f in toc_sections if f not in doc_files]
+ if len(files_not_exist) > 0:
+ message = "\n".join([f"- {f}" for f in files_not_exist])
+ raise RuntimeError(
+ "The following files are present in the table of contents but do not exist:\n"
+ + message
+ + f"\nRemove them from {toc_file}."
+ )
+
+ return sphinx_refs
+
+
+def convert_anchors_mapping_to_sphinx_format(anchors_mapping, package):
+ """
+ Convert the anchor mapping to the format expected by sphinx for the `objects.inv` file.
+
+ Args:
+ anchors_mapping (Dict[`str`, `str`]):
+ The mapping between anchors for objects in the doc and their location in the doc.
+ package (`types.ModuleType`):
+ The package in which to search objects for.
+ """
+ sphinx_refs = []
+ for anchor, url in anchors_mapping.items():
+ obj = find_object_in_package(anchor, package)
+ if isinstance(obj, property):
+ obj = obj.fget
+
+ # Object type
+ if isinstance(obj, type):
+ obj_type = "py:class"
+ elif hasattr(obj, "__name__") and hasattr(obj, "__qualname__"):
+ obj_type = "py:method" if obj.__name__ != obj.__qualname__ else "py:function"
+ else:
+ # Default to function (this part is never hit when building the docs for Transformers and Datasets)
+ # so it's just to be extra defensive
+ obj_type = "py:function"
+
+ if "#" in url:
+ sphinx_refs.append(f"{anchor} {obj_type} 1 {url} -")
+ else:
+ sphinx_refs.append(f"{anchor} {obj_type} 1 {url}#$ -")
+
+ return sphinx_refs
+
+
+def build_sphinx_objects_ref(sphinx_refs, output_dir, page_info):
+ """
+ Saves the sphinx references in an `objects.inv` file that can then be used by other documentations powered by
+ sphinx to link to objects in the generated doc.
+
+ Args:
+ sphinx_refs (`List[str]`): The list of all references, in the format expected by sphinx.
+ output_dir (`str` or `os.PathLike`): The folder where the doc is built.
+ page_info (`Dict[str, str]`): Some information about the doc.
+ """
+ intro = [
+ "# Sphinx inventory version 2\n",
+ f"# Project: {page_info['package_name']}\n",
+ f"# Version: {page_info['version']}\n",
+ "# The remainder of this file is compressed using zlib.\n",
+ ]
+ lines = [str.encode(line) for line in intro]
+
+ data = "\n".join(sorted(sphinx_refs)) + "\n"
+ data = zlib.compress(str.encode(data))
+
+ with open(Path(output_dir) / "objects.inv", "wb") as f:
+ f.writelines(lines)
+ f.write(data)
diff --git a/doc-build/third_party/hf-doc-builder/src/doc_builder/commands/__init__.py b/doc-build/third_party/hf-doc-builder/src/doc_builder/commands/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/doc-build/third_party/hf-doc-builder/src/doc_builder/commands/build.py b/doc-build/third_party/hf-doc-builder/src/doc_builder/commands/build.py
new file mode 100644
index 00000000..7bf09dd0
--- /dev/null
+++ b/doc-build/third_party/hf-doc-builder/src/doc_builder/commands/build.py
@@ -0,0 +1,222 @@
+# coding=utf-8
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import argparse
+import importlib
+import os
+import shutil
+import subprocess
+import tempfile
+from pathlib import Path
+
+from doc_builder import build_doc, update_versions_file
+from doc_builder.utils import get_default_branch_name, get_doc_config, locate_kit_folder, read_doc_config
+
+
+def check_node_is_available():
+ try:
+ p = subprocess.run(
+ ["node", "-v"],
+ stderr=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ check=True,
+ encoding="utf-8",
+ )
+ version = p.stdout.strip()
+ except Exception:
+ raise EnvironmentError(
+ "Using the --html flag requires node v14 to be installed, but it was not found in your system."
+ )
+
+ major = int(version[1:].split(".")[0])
+ if major < 14:
+ raise EnvironmentError(
+ "Using the --html flag requires node v14 to be installed, but the version in your system is lower "
+ f"({version[1:]})"
+ )
+
+
+def build_command(args):
+ read_doc_config(args.path_to_docs)
+ if args.html:
+ # Error at the beginning if node is not properly installed.
+ check_node_is_available()
+ # Error at the beginning if we can't locate the kit folder
+ kit_folder = locate_kit_folder()
+ if kit_folder is None:
+ raise EnvironmentError(
+ "Using the --html flag requires the kit subfolder of the doc-builder repo. We couldn't find it with "
+ "the doc-builder package installed, so you need to run the command from inside the doc-builder repo."
+ )
+
+ default_version = get_default_branch_name(args.path_to_docs)
+ if args.not_python_module and args.version is None:
+ version = default_version
+ elif args.version is None:
+ module = importlib.import_module(args.library_name)
+ version = module.__version__
+
+ if "dev" in version:
+ version = default_version
+ else:
+ version = f"v{version}"
+ else:
+ version = args.version
+
+ # `version` will always start with prefix `v`
+ # `version_tag` does not have to start with prefix `v` (see: https://github.com/huggingface/datasets/tags)
+ version_tag = version
+ if version != default_version:
+ doc_config = get_doc_config()
+ version_prefix = getattr(doc_config, "version_prefix", "v")
+ version_ = version[1:] # v2.1.0 -> 2.1.0
+ version_tag = f"{version_prefix}{version_}"
+
+ # Disable notebook building for non-master version
+ if version != default_version:
+ args.notebook_dir = None
+
+ notebook_dir = Path(args.notebook_dir) / args.language if args.notebook_dir is not None else None
+ output_path = Path(args.build_dir) / args.library_name / version / args.language
+
+ print("Building docs for", args.library_name, args.path_to_docs, output_path)
+ build_doc(
+ args.library_name,
+ args.path_to_docs,
+ output_path,
+ clean=args.clean,
+ version=version,
+ version_tag=version_tag,
+ language=args.language,
+ notebook_dir=notebook_dir,
+ is_python_module=not args.not_python_module,
+ version_tag_suffix=args.version_tag_suffix,
+ repo_owner=args.repo_owner,
+ repo_name=args.repo_name,
+ )
+
+ # dev build should not update _versions.yml
+ package_doc_path = os.path.join(args.build_dir, args.library_name)
+ if "pr_" not in version and os.path.isfile(os.path.join(package_doc_path, "_versions.yml")):
+ update_versions_file(os.path.join(args.build_dir, args.library_name), version, args.path_to_docs)
+
+ # If asked, convert the MDX files into HTML files.
+ if args.html:
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ tmp_dir = Path(tmp_dir)
+ # Copy everything in a tmp dir
+ shutil.copytree(kit_folder, tmp_dir / "kit")
+ # Manual copy and overwrite from output_path to tmp_dir / "kit" / "src" / "routes"
+ # We don't use shutil.copytree as tmp_dir / "kit" / "src" / "routes" exists and contains important files.
+ for f in output_path.iterdir():
+ dest = tmp_dir / "kit" / "src" / "routes" / f.name
+ if f.is_dir():
+ # Remove the dest folder if it exists
+ if dest.is_dir():
+ shutil.rmtree(dest)
+ shutil.copytree(f, dest)
+ else:
+ shutil.copy(f, dest)
+
+ # Move the objects.inv file at the root
+ if not args.not_python_module:
+ shutil.move(tmp_dir / "kit" / "src" / "routes" / "objects.inv", tmp_dir / "objects.inv")
+
+ # Build doc with node
+ working_dir = str(tmp_dir / "kit")
+ print("Installing node dependencies")
+ subprocess.run(
+ ["npm", "ci"],
+ stdout=subprocess.PIPE,
+ check=True,
+ encoding="utf-8",
+ cwd=working_dir,
+ )
+
+ env = os.environ.copy()
+ env["DOCS_LIBRARY"] = (
+ env["package_name"] or args.library_name if "package_name" in env else args.library_name
+ )
+ env["DOCS_VERSION"] = version
+ env["DOCS_LANGUAGE"] = args.language
+ print("Building HTML files. This will take a while :-)")
+ subprocess.run(
+ ["npm", "run", "build"],
+ stdout=subprocess.PIPE,
+ check=True,
+ encoding="utf-8",
+ cwd=working_dir,
+ env=env,
+ )
+
+ # Copy result back in the build_dir.
+ shutil.rmtree(output_path)
+ shutil.copytree(tmp_dir / "kit" / "build", output_path)
+ # Move the objects.inv file back
+ if not args.not_python_module:
+ shutil.move(tmp_dir / "objects.inv", output_path / "objects.inv")
+
+
+def build_command_parser(subparsers=None):
+ if subparsers is not None:
+ parser = subparsers.add_parser("build")
+ else:
+ parser = argparse.ArgumentParser("Doc Builder build command")
+
+ parser.add_argument("library_name", type=str, help="Library name")
+ parser.add_argument(
+ "path_to_docs",
+ type=str,
+ help="Local path to library documentation. The library should be cloned, and the folder containing the "
+ "documentation files should be indicated here.",
+ )
+ parser.add_argument("--build_dir", type=str, help="Where the built documentation will be.", default="./build/")
+ parser.add_argument("--clean", action="store_true", help="Whether or not to clean the output dir before building.")
+ parser.add_argument("--language", type=str, help="Language of the documentation to generate", default="en")
+ parser.add_argument(
+ "--version",
+ type=str,
+ help="Version of the documentation to generate. Will default to the version of the package module (using "
+ "`main` for a version containing dev).",
+ )
+ parser.add_argument("--notebook_dir", type=str, help="Where to save the generated notebooks.", default=None)
+ parser.add_argument("--html", action="store_true", help="Whether or not to build HTML files instead of MDX files.")
+ parser.add_argument(
+ "--not_python_module",
+ action="store_true",
+ help="Whether docs files do NOT have corresponding python module (like HF course & hub docs).",
+ )
+ parser.add_argument(
+ "--version_tag_suffix",
+ type=str,
+ default="src/",
+ help="Suffix to add after the version tag (e.g. 1.3.0 or main) in the documentation links. For example, the default `src/` suffix will result in a base link as `https://github.com/huggingface/{package_name}/blob/{version_tag}/src/`.",
+ )
+ parser.add_argument(
+ "--repo_owner",
+ type=str,
+ default="huggingface",
+ help="Owner of the repo (e.g. huggingface, rwightman, etc.).",
+ )
+ parser.add_argument(
+ "--repo_name",
+ type=str,
+ default=None,
+ help="Name of the repo (e.g. transformers, pytorch-image-models, etc.). By default, this is the same as the library_name.",
+ )
+ if subparsers is not None:
+ parser.set_defaults(func=build_command)
+ return parser
diff --git a/doc-build/third_party/hf-doc-builder/src/doc_builder/commands/convert_doc_file.py b/doc-build/third_party/hf-doc-builder/src/doc_builder/commands/convert_doc_file.py
new file mode 100644
index 00000000..8bf6ceee
--- /dev/null
+++ b/doc-build/third_party/hf-doc-builder/src/doc_builder/commands/convert_doc_file.py
@@ -0,0 +1,188 @@
+# coding=utf-8
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import argparse
+import re
+from pathlib import Path
+
+from doc_builder.autodoc import is_rst_docstring, remove_example_tags
+from doc_builder.convert_rst_to_mdx import (
+ apply_min_indent,
+ base_rst_to_mdx,
+ convert_rst_to_mdx,
+ find_indent,
+ is_empty_line,
+)
+
+
+def find_docstring_indent(docstring):
+ """
+ Finds the indent in the first nonempty line.
+ """
+ for line in docstring.split("\n"):
+ if not is_empty_line(line):
+ return find_indent(line)
+
+
+def find_root_git(folder):
+ "Finds the first parent folder who is a git directory or returns None if there is no git directory."
+ folder = Path(folder).absolute()
+ while folder != folder.parent and not (folder / ".git").exists():
+ folder = folder.parent
+ return folder if folder != folder.parent else None
+
+
+# Re pattern that matches links of the form [`some_class`]
+_re_internal_ref = re.compile("\[`([^`]*)`\]")
+
+
+def shorten_internal_refs(content):
+ """
+ Shortens links of the form [`~transformers.Trainer`] to just [`Trainer`].
+ """
+
+ def _shorten_ref(match):
+ full_name = match.groups()[0]
+ full_name = full_name.replace("transformers.", "")
+ if full_name.startswith("~") and "." not in full_name:
+ full_name = full_name[1:]
+ return f"[`{full_name}`]"
+
+ return _re_internal_ref.sub(_shorten_ref, content)
+
+
+def convert_rst_file(source_file, output_file, page_info):
+ with open(source_file, "r", encoding="utf-8") as f:
+ text = f.read()
+
+ text = convert_rst_to_mdx(text, page_info, add_imports=False)
+ text = text.replace("{", "{")
+ text = text.replace("<", "<")
+ text = re.sub(r"^\[\[autodoc\]\](\s+)(transformers\.)", r"[[autodoc]]\1", text, flags=re.MULTILINE)
+ text = shorten_internal_refs(text)
+
+ with open(output_file, "w", encoding="utf-8") as f:
+ f.write(text)
+
+
+def convert_rst_docstring_to_markdown(docstring, page_info):
+ """
+ Convert a given docstring in rst format to Markdown.
+ """
+ min_indent = find_docstring_indent(docstring)
+ docstring = base_rst_to_mdx(docstring, page_info, unindent=False)
+ docstring = remove_example_tags(docstring)
+ docstring = shorten_internal_refs(docstring)
+ docstring = apply_min_indent(docstring, min_indent)
+ docstring = docstring.replace("{", "{")
+ docstring = docstring.replace("<", "<")
+ return docstring
+
+
+def convert_rst_docstrings_in_file(source_file, output_file, page_info):
+ with open(source_file, "r", encoding="utf-8") as f:
+ code = f.read()
+ docstrings = code.split('"""')
+
+ for idx, docstring in enumerate(docstrings):
+ if idx % 2 == 0 or not is_rst_docstring(docstring):
+ continue
+ docstrings[idx] = convert_rst_docstring_to_markdown(docstring, page_info)
+
+ code = '"""'.join(docstrings)
+
+ with open(output_file, "w", encoding="utf-8") as f:
+ f.write(code)
+
+
+def convert_command(args):
+ source_file = Path(args.source_file).absolute()
+ if source_file.suffix not in [".rst", ".py"]:
+ raise ValueError(f"This script only converts rst files. Got {source_file}.")
+ if args.package_name is None:
+ git_folder = find_root_git(source_file)
+ if git_folder is None:
+ raise ValueError(
+ "Cannot determine a default for package_name as the file passed is not in a git directory. "
+ "Please pass along a package_name."
+ )
+ package_name = git_folder.name
+ else:
+ package_name = args.package_name
+
+ if args.output_file is None:
+ output_file = source_file.with_suffix(".mdx") if source_file.suffix == ".rst" else source_file
+ else:
+ output_file = args.output_file
+
+ page_info = {"package_name": package_name, "no_prefix": True}
+
+ if source_file.suffix == ".py":
+ convert_rst_docstrings_in_file(source_file, output_file, page_info)
+
+ else:
+ if args.doc_folder is None:
+ git_folder = find_root_git(source_file)
+ if git_folder is None:
+ raise ValueError(
+ "Cannot determine a default for package_name as the file passed is not in a git directory. "
+ "Please pass along a package_name."
+ )
+ doc_folder = (git_folder / "docs") / "source"
+ if doc_folder / source_file.relative_to(doc_folder) != source_file:
+ raise ValueError(
+ f"The default found for `doc_folder` is {doc_folder} but it does not look like {source_file} is "
+ "inside it."
+ )
+ else:
+ doc_folder = args.doc_folder
+
+ page_info["page"] = source_file.with_suffix(".html").relative_to(doc_folder)
+
+ convert_rst_file(source_file, output_file, page_info)
+
+
+def convert_command_parser(subparsers=None):
+ if subparsers is not None:
+ parser = subparsers.add_parser("convert")
+ else:
+ parser = argparse.ArgumentParser("Doc Builder convert command")
+
+ parser.add_argument("source_file", type=str, help="The file to convert.")
+ parser.add_argument(
+ "--package_name",
+ type=str,
+ default=None,
+ help="The name of the package this doc file belongs to. Will default to the name of the root git repo "
+ "`source_file` is in.",
+ )
+ parser.add_argument(
+ "--output_file",
+ type=str,
+ default=None,
+ help="Where to save the converted file. Will default to the `source_file` with an mdx suffix for rst files,"
+ "`source_file` for a py file.",
+ )
+ parser.add_argument(
+ "--doc_folder",
+ type=str,
+ help="The path to the folder with the doc source files. Will default to the `docs/source` subfolder of the "
+ "root git repo.",
+ )
+
+ if subparsers is not None:
+ parser.set_defaults(func=convert_command)
+ return parser
diff --git a/doc-build/third_party/hf-doc-builder/src/doc_builder/commands/doc_builder_cli.py b/doc-build/third_party/hf-doc-builder/src/doc_builder/commands/doc_builder_cli.py
new file mode 100644
index 00000000..e780414c
--- /dev/null
+++ b/doc-build/third_party/hf-doc-builder/src/doc_builder/commands/doc_builder_cli.py
@@ -0,0 +1,51 @@
+# coding=utf-8
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from argparse import ArgumentParser
+
+from doc_builder.commands.build import build_command_parser
+from doc_builder.commands.convert_doc_file import convert_command_parser
+from doc_builder.commands.notebook_to_mdx import notebook_to_mdx_command_parser
+from doc_builder.commands.preview import preview_command_parser
+from doc_builder.commands.push import push_command_parser
+from doc_builder.commands.style import style_command_parser
+
+
+def main():
+ parser = ArgumentParser("Doc Builder CLI tool", usage="doc-builder <command> [<args>]")
+ subparsers = parser.add_subparsers(help="doc-builder command helpers")
+
+ # Register commands
+ convert_command_parser(subparsers=subparsers)
+ build_command_parser(subparsers=subparsers)
+ notebook_to_mdx_command_parser(subparsers=subparsers)
+ style_command_parser(subparsers=subparsers)
+ preview_command_parser(subparsers=subparsers)
+ push_command_parser(subparsers=subparsers)
+
+ # Let's go
+ args = parser.parse_args()
+
+ if not hasattr(args, "func"):
+ parser.print_help()
+ exit(1)
+
+ # Run
+ args.func(args)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/doc-build/third_party/hf-doc-builder/src/doc_builder/commands/notebook_to_mdx.py b/doc-build/third_party/hf-doc-builder/src/doc_builder/commands/notebook_to_mdx.py
new file mode 100644
index 00000000..bea4dd10
--- /dev/null
+++ b/doc-build/third_party/hf-doc-builder/src/doc_builder/commands/notebook_to_mdx.py
@@ -0,0 +1,74 @@
+# coding=utf-8
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+
+import nbformat
+
+from ..style_doc import format_code_example
+
+
+def notebook_to_mdx_command(args):
+ notebook = nbformat.read(args.notebook_file, as_version=4)
+ content = []
+ for cell in notebook["cells"]:
+ if cell["cell_type"] == "code":
+ code = cell["source"]
+ outputs = [
+ o for o in cell["outputs"] if ("text" in o and o.get("name", None) == "stdout") or "text/plain" in o
+ ]
+ if len(outputs) > 0:
+ code_lines = code.split("\n")
+ # We can add >>> everywhere without worrying as format_code_examples will replace them by ...
+ # when needed.
+ code_lines = [f">>> {l}" if not len(l) == 0 or l.isspace() else l for l in code_lines]
+ code = "\n".join(code_lines)
+ code = format_code_example(code, max_len=args.max_len)[0]
+ content.append(f"```python\n{code}\n```")
+
+ output = outputs[0]["text"] if "text" in outputs[0] else outputs[0]["text/plain"]
+ output = output.strip()
+ content.append(f"```python out\n{output}\n```")
+ else:
+ code = format_code_example(code, max_len=args.max_len)[0]
+ content.append(f"```python\n{code}\n```")
+ elif cell["cell_type"] == "markdown":
+ content.append(cell["source"])
+ else:
+ content.append(f"```\n{cell['source']}\n```")
+
+ dest_file = args.dest_file if args.dest_file is not None else args.notebook_file.replace(".ipynb", ".mdx")
+ with open(dest_file, "w", encoding="utf-8") as f:
+ f.write("\n\n".join(content))
+
+
+def notebook_to_mdx_command_parser(subparsers=None):
+ if subparsers is not None:
+ parser = subparsers.add_parser("notebook-to-mdx")
+ else:
+ parser = argparse.ArgumentParser("Doc Builder convert notebook to MDX command")
+
+ parser.add_argument("notebook_file", type=str, help="The notebook to convert.")
+ parser.add_argument(
+ "--max_len",
+ type=int,
+ default=119,
+ help="The number of maximum characters per line.",
+ )
+ parser.add_argument("--dest_file", type=str, default=None, help="Where to save the result.")
+
+ if subparsers is not None:
+ parser.set_defaults(func=notebook_to_mdx_command)
+ return parser
diff --git a/doc-build/third_party/hf-doc-builder/src/doc_builder/commands/preview.py b/doc-build/third_party/hf-doc-builder/src/doc_builder/commands/preview.py
new file mode 100644
index 00000000..a50740db
--- /dev/null
+++ b/doc-build/third_party/hf-doc-builder/src/doc_builder/commands/preview.py
@@ -0,0 +1,237 @@
+# coding=utf-8
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import argparse
+import os
+import shutil
+import subprocess
+import tempfile
+import time
+from pathlib import Path
+from threading import Thread
+
+from doc_builder import build_doc
+from doc_builder.commands.build import check_node_is_available, locate_kit_folder
+from doc_builder.commands.convert_doc_file import find_root_git
+from doc_builder.utils import is_watchdog_available, read_doc_config
+
+
+if is_watchdog_available():
+ from watchdog.events import FileSystemEventHandler
+ from watchdog.observers import Observer
+
+ class WatchEventHandler(FileSystemEventHandler):
+ """
+ Utility class for building updated mdx files when a file change event is recorded.
+ """
+
+ def __init__(self, args, source_files_mapping, kit_routes_folder):
+ super().__init__()
+ self.args = args
+ self.source_files_mapping = source_files_mapping
+ self.kit_routes_folder = kit_routes_folder
+
+ def on_created(self, event):
+ super().on_created(event)
+ is_valid, src_path, relative_path = self.transform_path(event)
+ if is_valid:
+ self.build(src_path, relative_path)
+
+ def on_modified(self, event):
+ super().on_modified(event)
+ is_valid, src_path, relative_path = self.transform_path(event)
+ if is_valid:
+ self.build(src_path, relative_path)
+
+ def transform_path(self, event):
+ """
+ Check if a file is a doc file (mdx, or py file used as autodoc).
+ If so, returns mdx file path.
+ """
+ src_path = event.src_path
+ parent_path_absolute = str(Path(self.args.path_to_docs).absolute())
+ relative_path = event.src_path[len(parent_path_absolute) + 1 :]
+ is_valid_file = False
+ if not event.is_directory:
+ if src_path.endswith(".py") and src_path in self.source_files_mapping:
+ src_path = self.source_files_mapping[src_path]
+ # if src_path.endswith(".md"):
+ # # src_path += "x"
+ # relative_path += "x"
+ if src_path.endswith(".mdx") or src_path.endswith(".md"):
+ is_valid_file = True
+ return is_valid_file, src_path, relative_path
+ return is_valid_file, src_path, relative_path
+
+ def build(self, src_path, relative_path):
+ """
+ Build single mdx file in a temp dir.
+ """
+ print(f"Building: {src_path}")
+ try:
+ # copy the built files into the actual build folder dawg
+ with tempfile.TemporaryDirectory() as tmp_input_dir:
+ # copy the file into tmp_input_dir
+ shutil.copy(src_path, tmp_input_dir)
+
+ with tempfile.TemporaryDirectory() as tmp_out_dir:
+ build_doc(
+ self.args.library_name,
+ tmp_input_dir,
+ tmp_out_dir,
+ version=self.args.version,
+ language=self.args.language,
+ is_python_module=not self.args.not_python_module,
+ watch_mode=True,
+ )
+ if str(src_path).endswith(".md"):
+ src_path += "x"
+ relative_path += "x"
+ src = Path(tmp_out_dir) / Path(src_path).name
+ dest = self.kit_routes_folder / relative_path
+ shutil.move(src, dest)
+ except Exception as e:
+ print(f"Error building: {src_path}\n{e}")
+
+
+def start_watcher(path, event_handler):
+ """
+ Starts `pywatchdog.observer` for listening changes in `path`.
+ """
+ observer = Observer()
+ observer.schedule(event_handler, path, recursive=True)
+ observer.start()
+ print(f"\nWatching for changes in: {path}\n")
+ try:
+ while True:
+ time.sleep(1)
+ except KeyboardInterrupt:
+ observer.stop()
+ observer.join()
+
+
+def start_sveltekit_dev(tmp_dir, env, args):
+ """
+ Installs sveltekit node dependencies & starts sveltekit in dev mode in a temp dir.
+ """
+ working_dir = str(tmp_dir / "kit")
+ print("Installing node dependencies")
+ subprocess.run(
+ ["npm", "ci"],
+ stdout=subprocess.PIPE,
+ check=True,
+ encoding="utf-8",
+ cwd=working_dir,
+ )
+
+ # start sveltekit in dev mode
+ subprocess.run(
+ ["npm", "run", "dev"],
+ check=True,
+ encoding="utf-8",
+ cwd=working_dir,
+ env=env,
+ )
+
+
+def preview_command(args):
+ if not is_watchdog_available():
+ raise ImportError(
+ "Please install `watchdog` to run `doc-builder preview` command.\nYou can do so through pip: `pip install watchdog`"
+ )
+
+ read_doc_config(args.path_to_docs)
+ # Error at the beginning if node is not properly installed.
+ check_node_is_available()
+ # Error at the beginning if we can't locate the kit folder
+ kit_folder = locate_kit_folder()
+ if kit_folder is None:
+ raise EnvironmentError(
+ "Requires the kit subfolder of the doc-builder repo. We couldn't find it with "
+ "the doc-builder package installed, so you need to run the command from inside the doc-builder repo."
+ )
+
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ output_path = Path(tmp_dir) / args.library_name / args.version / args.language
+
+ print("Initial build docs for", args.library_name, args.path_to_docs, output_path)
+ source_files_mapping = build_doc(
+ args.library_name,
+ args.path_to_docs,
+ output_path,
+ clean=True,
+ version=args.version,
+ language=args.language,
+ is_python_module=not args.not_python_module,
+ )
+
+ # convert the MDX files into HTML files.
+ tmp_dir = Path(tmp_dir)
+ # Copy everything in a tmp dir
+ shutil.copytree(kit_folder, tmp_dir / "kit")
+ # Manual copy and overwrite from output_path to tmp_dir / "kit" / "src" / "routes"
+ # We don't use shutil.copytree as tmp_dir / "kit" / "src" / "routes" exists and contains important files.
+ kit_routes_folder = tmp_dir / "kit" / "src" / "routes"
+ # files/folders cannot have a name that starts with `__` since it is a reserved Sveltekit keyword
+ for p in output_path.glob("**/*__*"):
+ if p.exists():
+ p.rmdir if p.is_dir() else p.unlink()
+ for f in output_path.iterdir():
+ dest = kit_routes_folder / f.name
+ if f.is_dir():
+ # Remove the dest folder if it exists
+ if dest.is_dir():
+ shutil.rmtree(dest)
+ shutil.copytree(f, dest)
+ else:
+ shutil.copy(f, dest)
+
+ # Node
+ env = os.environ.copy()
+ env["DOCS_LIBRARY"] = env["package_name"] or args.library_name if "package_name" in env else args.library_name
+ env["DOCS_VERSION"] = args.version
+ env["DOCS_LANGUAGE"] = args.language
+ Thread(target=start_sveltekit_dev, args=(tmp_dir, env, args)).start()
+
+ git_folder = find_root_git(args.path_to_docs)
+ event_handler = WatchEventHandler(args, source_files_mapping, kit_routes_folder)
+ start_watcher(git_folder, event_handler)
+
+
+def preview_command_parser(subparsers=None):
+ if subparsers is not None:
+ parser = subparsers.add_parser("preview")
+ else:
+ parser = argparse.ArgumentParser("Doc Builder preview command")
+
+ parser.add_argument("library_name", type=str, help="Library name")
+ parser.add_argument(
+ "path_to_docs",
+ type=str,
+ help="Local path to library documentation. The library should be cloned, and the folder containing the "
+ "documentation files should be indicated here.",
+ )
+ parser.add_argument("--language", type=str, help="Language of the documentation to generate", default="en")
+ parser.add_argument("--version", type=str, help="Version of the documentation to generate", default="main")
+ parser.add_argument(
+ "--not_python_module",
+ action="store_true",
+ help="Whether docs files do NOT have corresponding python module (like HF course & hub docs).",
+ )
+
+ if subparsers is not None:
+ parser.set_defaults(func=preview_command)
+ return parser
diff --git a/doc-build/third_party/hf-doc-builder/src/doc_builder/commands/push.py b/doc-build/third_party/hf-doc-builder/src/doc_builder/commands/push.py
new file mode 100644
index 00000000..e4e83abe
--- /dev/null
+++ b/doc-build/third_party/hf-doc-builder/src/doc_builder/commands/push.py
@@ -0,0 +1,188 @@
+# coding=utf-8
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import logging
+import os
+import shutil
+from pathlib import Path
+from time import sleep, time
+
+from huggingface_hub import HfApi
+
+
+REPO_TYPE = "dataset"
+SEPARATOR = "/"
+
+
+def create_zip_name(library_name, version, with_ext=True):
+ file_name = f"{library_name}{SEPARATOR}{version}"
+ if with_ext:
+ file_name += ".zip"
+ return file_name
+
+
+def push_command(args):
+ """
+ Commit file doc builds changes using: 1. zip doc build artifacts 2. hf_hub client to upload/delete zip file
+ Usage: doc-builder push $args
+ """
+ if args.n_retries < 1:
+ raise ValueError(f"CLI arg `n_retries` MUST be positive & non-zero; supplied value was {args.n_retries}")
+ if args.upload_version_yml:
+ library_name = args.library_name
+ version_file_path = f"{library_name}/_versions.yml"
+ api = HfApi()
+ api.upload_file(
+ repo_id=args.doc_build_repo_id,
+ repo_type=REPO_TYPE,
+ path_or_fileobj=version_file_path,
+ path_in_repo=version_file_path,
+ commit_message="Updating _version.yml",
+ token=args.token,
+ )
+ os.remove(version_file_path)
+ if args.is_remove:
+ push_command_remove(args)
+ else:
+ push_command_add(args)
+
+
+def push_command_add(args):
+ """
+ Commit file changes using: 1. zip doc build artifacts 2. hf_hub client to upload zip file
+ Used in: build_main_documentation.yml & build_pr_documentation.yml
+ """
+ max_n_retries = args.n_retries + 1
+ number_of_retries = args.n_retries
+ n_seconds_sleep = 5
+
+ library_name = args.library_name
+ path_docs_built = Path(library_name)
+ doc_version_folder = next(filter(lambda x: not x.is_file(), path_docs_built.glob("*")), None).relative_to(
+ path_docs_built
+ )
+ doc_version_folder = str(doc_version_folder)
+
+ zip_file_path_without_ext = create_zip_name(library_name, doc_version_folder, with_ext=False)
+ # zips current dir. In this case, doc-build dir
+ shutil.make_archive(zip_file_path_without_ext, "zip")
+ zip_file_path = create_zip_name(library_name, doc_version_folder)
+
+ api = HfApi()
+
+ time_start = time()
+
+ while number_of_retries:
+ try:
+ api.upload_file(
+ repo_id=args.doc_build_repo_id,
+ repo_type=REPO_TYPE,
+ path_or_fileobj=zip_file_path,
+ path_in_repo=zip_file_path,
+ commit_message=args.commit_msg,
+ token=args.token,
+ )
+ break
+ except Exception as e:
+ number_of_retries -= 1
+ print(f"push_command_add error occurred: {e}")
+ if number_of_retries:
+ print(f"Failed on try #{max_n_retries-number_of_retries}, pushing again in {n_seconds_sleep} seconds")
+ sleep(n_seconds_sleep)
+ else:
+ raise RuntimeError("push_command_add failed") from e
+
+ time_end = time()
+ logging.debug(f"push_command_add took {time_end-time_start:.4f} seconds or {(time_end-time_start)/60.0:.2f} mins")
+
+
+def push_command_remove(args):
+ """
+ Commit file deletions using hf_hub client to delete zip file
+ Used in: delete_doc_comment.yml
+ """
+ max_n_retries = args.n_retries + 1
+ number_of_retries = args.n_retries
+ n_seconds_sleep = 5
+
+ library_name = args.library_name
+ doc_version_folder = args.doc_version
+ doc_build_repo_id = args.doc_build_repo_id
+ commit_msg = args.commit_msg
+
+ api = HfApi()
+ zip_file_path = create_zip_name(library_name, doc_version_folder)
+
+ while number_of_retries:
+ try:
+ api.delete_file(
+ zip_file_path, doc_build_repo_id, token=args.token, repo_type=REPO_TYPE, commit_message=commit_msg
+ )
+ break
+ except Exception as e:
+ number_of_retries -= 1
+ print(f"push_command_remove error occurred: {e}")
+ if number_of_retries:
+ print(f"Failed on try #{max_n_retries-number_of_retries}, pushing again in {n_seconds_sleep} seconds")
+ sleep(n_seconds_sleep)
+ else:
+ raise RuntimeError("push_command_remove failed") from e
+
+
+def push_command_parser(subparsers=None):
+ if subparsers is not None:
+ parser = subparsers.add_parser("push")
+ else:
+ parser = argparse.ArgumentParser("Doc Builder push command")
+
+ parser.add_argument(
+ "library_name",
+ type=str,
+ help="The name of the library, which also acts as a path where built doc artifacts reside in",
+ )
+ parser.add_argument(
+ "--doc_build_repo_id",
+ type=str,
+ help="Repo to which doc artifacts will be committed (e.g. `huggingface/doc-build-dev`)",
+ )
+ parser.add_argument("--token", type=str, help="Github token that has write/push permission to `doc_build_repo_id`")
+ parser.add_argument(
+ "--commit_msg",
+ type=str,
+ help="Git commit message",
+ default="Github GraphQL createcommitonbranch commit",
+ )
+ parser.add_argument("--n_retries", type=int, help="Number of push retries in the event of conflict", default=1)
+ parser.add_argument(
+ "--doc_version",
+ type=str,
+ default=None,
+ help="Version of the generated documentation.",
+ )
+ parser.add_argument(
+ "--is_remove",
+ action="store_true",
+ help="Whether or not to remove entire folder ('--doc_version') from git tree",
+ )
+ parser.add_argument(
+ "--upload_version_yml",
+ action="store_true",
+ help="Whether or not to push _version.yml file to git repo",
+ )
+
+ if subparsers is not None:
+ parser.set_defaults(func=push_command)
+ return parser
diff --git a/doc-build/third_party/hf-doc-builder/src/doc_builder/commands/style.py b/doc-build/third_party/hf-doc-builder/src/doc_builder/commands/style.py
new file mode 100644
index 00000000..b44c06e1
--- /dev/null
+++ b/doc-build/third_party/hf-doc-builder/src/doc_builder/commands/style.py
@@ -0,0 +1,46 @@
+# coding=utf-8
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import argparse
+
+from doc_builder import style_doc_files
+from doc_builder.utils import read_doc_config
+
+
+def style_command(args):
+ if args.path_to_docs is not None:
+ read_doc_config(args.path_to_docs)
+ changed = style_doc_files(*args.files, max_len=args.max_len, check_only=args.check_only)
+ if args.check_only and len(changed) > 0:
+ raise ValueError(f"{len(changed)} files should be restyled!")
+ elif len(changed) > 0:
+ print(f"Cleaned {len(changed)} files!")
+
+
+def style_command_parser(subparsers=None):
+ if subparsers is not None:
+ parser = subparsers.add_parser("style")
+ else:
+ parser = argparse.ArgumentParser("Doc Builder style command")
+
+ parser.add_argument("files", nargs="+", help="The file(s) or folder(s) to restyle.")
+ parser.add_argument("--path_to_docs", type=str, help="The path to the doc source folder if using the config.")
+ parser.add_argument("--max_len", type=int, default=119, help="The maximum length of lines.")
+ parser.add_argument("--check_only", action="store_true", help="Whether to only check and not fix styling issues.")
+
+ if subparsers is not None:
+ parser.set_defaults(func=style_command)
+ return parser
diff --git a/doc-build/third_party/hf-doc-builder/src/doc_builder/convert_md_to_mdx.py b/doc-build/third_party/hf-doc-builder/src/doc_builder/convert_md_to_mdx.py
new file mode 100644
index 00000000..e52b42b9
--- /dev/null
+++ b/doc-build/third_party/hf-doc-builder/src/doc_builder/convert_md_to_mdx.py
@@ -0,0 +1,178 @@
+# coding=utf-8
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import json
+import re
+import tempfile
+
+from .convert_rst_to_mdx import parse_rst_docstring, remove_indent
+
+
+_re_doctest_flags = re.compile("^(>>>.*\S)(\s+)# doctest:\s+\+[A-Z_]+\s*$", flags=re.MULTILINE)
+
+
+def convert_md_to_mdx(md_text, page_info):
+ """
+ Convert a document written in md to mdx.
+ """
+ return """<script lang="ts">
+import {onMount} from "svelte";
+import Tip from "$lib/Tip.svelte";
+import Youtube from "$lib/Youtube.svelte";
+import Docstring from "$lib/Docstring.svelte";
+import CodeBlock from "$lib/CodeBlock.svelte";
+import CodeBlockFw from "$lib/CodeBlockFw.svelte";
+import DocNotebookDropdown from "$lib/DocNotebookDropdown.svelte";
+import CourseFloatingBanner from "$lib/CourseFloatingBanner.svelte";
+import IconCopyLink from "$lib/IconCopyLink.svelte";
+import FrameworkContent from "$lib/FrameworkContent.svelte";
+import Markdown from "$lib/Markdown.svelte";
+import Question from "$lib/Question.svelte";
+import FrameworkSwitchCourse from "$lib/FrameworkSwitchCourse.svelte";
+import InferenceApi from "$lib/InferenceApi.svelte";
+import TokenizersLanguageContent from "$lib/TokenizersLanguageContent.svelte";
+import ExampleCodeBlock from "$lib/ExampleCodeBlock.svelte";
+import Added from "$lib/Added.svelte";
+import Changed from "$lib/Changed.svelte";
+import Deprecated from "$lib/Deprecated.svelte";
+import PipelineIcon from "$lib/PipelineIcon.svelte";
+import PipelineTag from "$lib/PipelineTag.svelte";
+let fw: "pt" | "tf" = "pt";
+onMount(() => {
+ const urlParams = new URLSearchParams(window.location.search);
+ fw = urlParams.get("fw") || "pt";
+});
+</script>
+<svelte:head>
+ <meta name="hf:doc:metadata" content={JSON.stringify(metadata)} >
+</svelte:head>
+""" + process_md(
+ md_text, page_info
+ )
+
+
+def convert_special_chars(text):
+ """
+ Convert { and < that have special meanings in MDX.
+ """
+ _re_lcub_svelte = re.compile(
+ r"<(Question|Tip|Added|Changed|Deprecated|DocNotebookDropdown|CourseFloatingBanner|FrameworkSwitch|audio|PipelineIcon|PipelineTag)(((?!<(Question|Tip|Added|Changed|Deprecated|DocNotebookDropdown|CourseFloatingBanner|FrameworkSwitch|audio|PipelineIcon|PipelineTag)).)*)>|{(#if|:else}|/if})",
+ re.DOTALL,
+ )
+ text = text.replace("{", "{")
+ # We don't want to escape `{` that are part of svelte syntax
+ text = _re_lcub_svelte.sub(lambda match: match[0].replace("{", "{"), text)
+ # We don't want to replace those by the HTML code, so we temporarily set them at LTHTML
+ text = re.sub(
+ r"<(img|video|br|hr|Youtube|Question|DocNotebookDropdown|CourseFloatingBanner|FrameworkSwitch|audio|PipelineIcon|PipelineTag)",
+ r"LTHTML\1",
+ text,
+ ) # html void elements with no closing counterpart
+ _re_lt_html = re.compile(r"<(\S+)([^>]*>)(((?!</\1>).)*)<(/\1>)", re.DOTALL)
+ while _re_lt_html.search(text):
+ text = _re_lt_html.sub(r"LTHTML\1\2\3LTHTML\5", text)
+ text = re.sub(r"(^|[^<])<([^(<|!)]|$)", r"\1<\2", text)
+ text = text.replace("LTHTML", "<")
+ return text
+
+
+def convert_img_links(text, page_info):
+ """
+ Convert image links to correct URL paths.
+ """
+ if "package_name" not in page_info:
+ raise ValueError("`page_info` must contain at least the package_name.")
+ package_name = page_info["package_name"]
+ version = page_info.get("version", "main")
+ language = page_info.get("language", "en")
+
+ _re_img_link = re.compile(r"(src=\"|\()/imgs/")
+ while _re_img_link.search(text):
+ text = _re_img_link.sub(rf"\1/docs/{package_name}/{version}/{language}/imgs/", text)
+ return text
+
+
+def clean_doctest_syntax(text):
+ """
+ Clean the doctest artifacts in a given content.
+ """
+ text = text.replace(">>> # ===PT-TF-SPLIT===", "===PT-TF-SPLIT===")
+ text = _re_doctest_flags.sub(r"\1", text)
+ return text
+
+
+_re_literalinclude = re.compile(r"([ \t]*)<literalinclude>(((?!<literalinclude>).)*)<\/literalinclude>", re.DOTALL)
+
+
+def convert_literalinclude_helper(match, page_info):
+ """
+ Convert a literalinclude regex match into markdown code blocks by opening a file and
+ copying specified start-end section into markdown code block.
+ """
+ literalinclude_info = json.loads(match[2].strip())
+ indent = match[1]
+ if tempfile.gettempdir() in str(page_info["path"]):
+ return "\n`Please restart doc-builder preview commands to see literalinclude rendered`\n"
+ file = page_info["path"].parent / literalinclude_info["path"]
+ with open(file, "r", encoding="utf-8-sig") as reader:
+ lines = reader.readlines()
+ literalinclude = lines # defaults to entire file
+ if "start-after" in literalinclude_info or "end-before" in literalinclude_info:
+ start_after, end_before = -1, -1
+ for idx, line in enumerate(lines):
+ line = line.strip()
+ line = re.sub(r"\W+$", "", line)
+ if line.endswith(literalinclude_info["start-after"]):
+ start_after = idx + 1
+ if line.endswith(literalinclude_info["end-before"]):
+ end_before = idx
+ if start_after == -1 or end_before == -1:
+ raise ValueError(f"The following 'literalinclude' does NOT exist:\n{match[0]}")
+ literalinclude = lines[start_after:end_before]
+ literalinclude = [indent + line[literalinclude_info.get("dedent", 0) :] for line in literalinclude]
+ literalinclude = "".join(literalinclude)
+ return f"""{indent}```{literalinclude_info.get('language', '')}\n{literalinclude.rstrip()}\n{indent}```"""
+
+
+def convert_literalinclude(text, page_info):
+ """
+ Convert a literalinclude into markdown code blocks.
+ """
+ text = _re_literalinclude.sub(lambda m: convert_literalinclude_helper(m, page_info), text)
+ return text
+
+
+def convert_md_docstring_to_mdx(docstring, page_info):
+ """
+ Convert a docstring written in Markdown to mdx.
+ """
+ text = parse_rst_docstring(docstring)
+ text = remove_indent(text)
+ return process_md(text, page_info)
+
+
+def process_md(text, page_info):
+ """
+ Processes markdown by:
+ 1. Converting literalinclude
+ 2. Converting special characters
+ 3. Converting image links
+ """
+ text = convert_literalinclude(text, page_info)
+ text = convert_special_chars(text)
+ text = clean_doctest_syntax(text)
+ text = convert_img_links(text, page_info)
+ return text
diff --git a/doc-build/third_party/hf-doc-builder/src/doc_builder/convert_rst_to_mdx.py b/doc-build/third_party/hf-doc-builder/src/doc_builder/convert_rst_to_mdx.py
new file mode 100644
index 00000000..84551d79
--- /dev/null
+++ b/doc-build/third_party/hf-doc-builder/src/doc_builder/convert_rst_to_mdx.py
@@ -0,0 +1,705 @@
+# coding=utf-8
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import re
+
+
+# Re pattern to catch things inside ` ` in :obj:`thing`.
+_re_obj = re.compile(r":obj:`([^`]+)`")
+# Re pattern to catch things inside ` ` in :math:`thing`.
+_re_math = re.compile(r":math:`([^`]+)`")
+# Re pattern to catch things between single backquotes.
+_re_single_backquotes = re.compile(r"(^|[^`])`([^`]+)`([^`]|$)")
+# Re pattern to catch things between double backquotes.
+_re_double_backquotes = re.compile(r"(^|[^`])``([^`]+)``([^`]|$)")
+# Re pattern to catch things inside ` ` in :func/class/meth:`thing`.
+_re_func_class = re.compile(r":(?:func|class|meth):`([^`]+)`")
+
+
+def convert_rst_formatting(text):
+ """
+ Convert rst syntax for formatting to markdown in a given text.
+ """
+ # Remove :class:, :func: and :meth: markers. To code-links and put double backquotes
+ # (to not be caught by the italic conversion).
+ text = _re_func_class.sub(r"[``\1``]", text)
+ # Remove :obj: markers. What's after is in a single backquotes so we put in double backquotes
+ # (to not be caught by the italic conversion).
+ text = _re_obj.sub(r"``\1``", text)
+ # Remove :math: markers.
+ text = _re_math.sub(r"\\\\(\1\\\\)", text)
+ # Convert content in single backquotes to italic.
+ text = _re_single_backquotes.sub(r"\1*\2*\3", text)
+ # Convert content in double backquotes to single backquotes.
+ text = _re_double_backquotes.sub(r"\1`\2`\3", text)
+ # Remove remaining ::
+ text = re.sub(r"::\n", "", text)
+
+ # Remove new lines inside blocks in backsticks as they will be kept.
+ lines = text.split("\n")
+ in_code = False
+ text = None
+ for line in lines:
+ if in_code:
+ splits = line.split("`")
+ in_code = len(splits) > 1 and len(splits) % 2 == 1
+ if len(splits) == 1:
+ # Some forgotten lone backstick
+ text += "\n" + line
+ else:
+ text += " " + line.lstrip()
+ else:
+ if text is not None:
+ text += "\n" + line
+ else:
+ text = line
+ splits = line.split("`")
+ in_code = len(splits) % 2 == 0
+ return text
+
+
+# Re pattern to catch description and url in links of the form `description <url>`_.
+_re_links = re.compile(r"`([^`]+\S)\s+</*([^/][^>`]*)>`_+")
+# Re pattern to catch description and url in links of the form :prefix_link:`description <url>`_.
+_re_prefix_links = re.compile(r":prefix_link:`([^`]+\S)\s+</*([^/][^>`]*)>`")
+# Re pattern to catch reference in links of the form :doc:`reference`.
+_re_simple_doc = re.compile(r":doc:`([^`<]*)`")
+# Re pattern to catch description and reference in links of the form :doc:`description <reference>`.
+_re_doc_with_description = re.compile(r":doc:`([^`<]+\S)\s+</*([^/][^>`]*)>`")
+# Re pattern to catch reference in links of the form :ref:`reference`.
+_re_simple_ref = re.compile(r":ref:`([^`<]*)`")
+# Re pattern to catch description and reference in links of the form :ref:`description <reference>`.
+_re_ref_with_description = re.compile(r":ref:`([^`<]+\S)\s+<([^>]*)>`")
+
+
+def convert_rst_links(text, page_info):
+ """
+ Convert the rst links in text to markdown.
+ """
+ if "package_name" not in page_info:
+ raise ValueError("`page_info` must contain at least the package_name.")
+ package_name = page_info["package_name"]
+ version = page_info.get("version", "main")
+ language = page_info.get("language", "en")
+ no_prefix = page_info.get("no_prefix", False)
+
+ prefix = "" if no_prefix else f"/docs/{package_name}/{version}/{language}/"
+ # Links of the form :doc:`page`
+ text = _re_simple_doc.sub(rf"[\1]({prefix}\1)", text)
+ # Links of the form :doc:`text <page>`
+ text = _re_doc_with_description.sub(rf"[\1]({prefix}\2)", text)
+
+ if "page" in page_info and not no_prefix:
+ page = str(page_info["page"])
+ if page.endswith(".html"):
+ page = page[:-5]
+ prefix = f"{prefix}{page}"
+ else:
+ prefix = ""
+ # Refs of the form :ref:`page`
+ text = _re_simple_ref.sub(rf"[\1]({prefix}#\1)", text)
+ # Refs of the form :ref:`text <page>`
+ text = _re_ref_with_description.sub(rf"[\1]({prefix}#\2)", text)
+
+ # Links with a prefix
+ # TODO: when it exists, use the API to deal with prefix links properly.
+ prefix = f"https://github.com/huggingface/{package_name}/tree/main/"
+ text = _re_prefix_links.sub(rf"[\1]({prefix}\2)", text)
+ # Other links
+ text = _re_links.sub(r"[\1](\2)", text)
+ # Relative links or Transformers links need to remove the .html
+ if "(https://https://huggingface.co/" in text or re.search("\(\.+/", text) is not None:
+ text = text.replace(".html", "")
+ return text
+
+
+# Re pattern that catches examples blocks of the form `Example::`.
+_re_example = re.compile("^\s*(\S.*)::\s*$")
+# Re pattern that catches rst blocks of the form `.. block_name::`.
+_re_block = re.compile(r"^\s*\.\.\s+(\S+)::")
+# Re pattern that catches what's after the :: in rst blocks of the form `.. block_name:: something`.
+_re_block_info = re.compile(r"^\s*\.\.\s+\S+::\s*(\S.*)$")
+
+
+def is_empty_line(line):
+ return len(line) == 0 or line.isspace()
+
+
+def find_indent(line):
+ """
+ Returns the number of spaces that start a line indent.
+ """
+ search = re.search("^(\s*)(?:\S|$)", line)
+ if search is None:
+ return 0
+ return len(search.groups()[0])
+
+
+_re_rst_option = re.compile("^\s*:(\S+):(.*)$")
+
+
+def convert_special_chars(text):
+ """
+ Converts { and < that have special meanings in MDX.
+ """
+ text = text.replace("{", "{")
+ # We don't want to replace those by the HTML code, so we temporarily set them at LTHTML
+ text = re.sub(r"<(img|br|hr|Youtube)", r"LTHTML\1", text) # html void elements with no closing counterpart
+ _re_lt_html = re.compile(r"<(\S+)([^>]*>)(((?!</\1>).)*)<(/\1>)", re.DOTALL)
+ while _re_lt_html.search(text):
+ text = _re_lt_html.sub(r"LTHTML\1\2\3LTHTML\5", text)
+ text = re.sub(r"(^|[^<])<([^<]|$)", r"\1<\2", text)
+ text = text.replace("LTHTML", "<")
+ return text
+
+
+def parse_options(block_content):
+ """
+ Parses the option in some rst block content.
+ """
+ block_lines = block_content.split("\n")
+ block_indent = find_indent(block_lines[0])
+ current_option = None
+ result = {}
+ for line in block_lines:
+ if _re_rst_option.search(line) is not None:
+ current_option, value = _re_rst_option.search(line).groups()
+ result[current_option] = value.lstrip()
+ elif find_indent(line) > block_indent:
+ result[current_option] += " " + line.lstrip()
+
+ return result
+
+
+def apply_min_indent(text, min_indent):
+ """
+ Make sure all lines in a text are have a minimum indentation.
+
+ Args:
+ text (`str`): The text to treat.
+ min_indent (`int`): The minimal indentation.
+
+ Returns:
+ `str`: The processed text.
+ """
+ lines = text.split("\n")
+ idx = 0
+ while idx < len(lines):
+ if is_empty_line(lines[idx]):
+ idx += 1
+ continue
+ indent = find_indent(lines[idx])
+ if indent < min_indent:
+ while idx < len(lines) and (find_indent(lines[idx]) >= indent or is_empty_line(lines[idx])):
+ if not is_empty_line(lines[idx]):
+ lines[idx] = " " * (min_indent - indent) + lines[idx]
+ idx += 1
+ else:
+ idx += 1
+
+ return "\n".join(lines)
+
+
+def convert_rst_blocks(text, page_info):
+ """
+ Converts rst special blocks (examples, notes) into MDX.
+ """
+ if "package_name" not in page_info:
+ raise ValueError("`page_info` must contain at least the package_name.")
+ package_name = page_info["package_name"]
+ version = page_info.get("version", "main")
+ language = page_info.get("language", "en")
+
+ lines = text.split("\n")
+ idx = 0
+ new_lines = []
+ while idx < len(lines):
+ block_type = None
+ block_info = None
+ if _re_block.search(lines[idx]) is not None:
+ block_type = _re_block.search(lines[idx]).groups()[0]
+ if _re_block_info.search(lines[idx]) is not None:
+ block_info = _re_block_info.search(lines[idx]).groups()[0]
+ elif _re_example.search(lines[idx]) is not None:
+ block_type = "code-block-example"
+ block_info = "python"
+ example_name = _re_example.search(lines[idx]).groups()[0]
+ new_lines.append(f"<exampletitle>{example_name}:</exampletitle>\n")
+ elif lines[idx].strip() == "..":
+ block_type = "comment"
+ elif lines[idx].strip() == "::":
+ block_type = "code-block"
+
+ if block_type is not None:
+ block_indent = find_indent(lines[idx])
+ # Find the next nonempty line
+ idx += 1
+ while idx < len(lines) and is_empty_line(lines[idx]):
+ idx += 1
+ # Grab the indent of the return line, this block will stop when we unindent under it (or has already)
+ example_indent = find_indent(lines[idx]) if idx < len(lines) else block_indent
+
+ if example_indent == block_indent:
+ block_content = ""
+ else:
+ block_lines = []
+ while idx < len(lines) and (is_empty_line(lines[idx]) or find_indent(lines[idx]) >= example_indent):
+ block_lines.append(lines[idx][example_indent:])
+ idx += 1
+ block_content = "\n".join(block_lines)
+
+ if block_type in ["code", "code-block"]:
+ prefix = "```" if block_info is None else f"```{block_info}"
+ new_lines.append(f"{prefix}\n{block_content.strip()}\n```\n")
+ elif block_type == "code-block-example":
+ prefix = f"<example>```{block_info}"
+ new_lines.append(f"{prefix}\n{block_content.strip()}\n```\n</example>")
+ elif block_type == "note":
+ new_lines.append(apply_min_indent(f"<Tip>\n\n{block_content.strip()}\n\n</Tip>\n", block_indent))
+ elif block_type == "warning":
+ new_lines.append(
+ apply_min_indent("<Tip warning={true}>\n\n" + f"{block_content.strip()}\n\n</Tip>\n", block_indent)
+ )
+ elif block_type == "raw":
+ new_lines.append(block_content.strip() + "\n")
+ elif block_type == "math":
+ new_lines.append(f"$${block_content.strip()}$$\n")
+ elif block_type == "comment":
+ new_lines.append(f"<!--{block_content.strip()}\n-->\n")
+ elif block_type == "autofunction":
+ if block_info is not None:
+ new_lines.append(f"[[autodoc]] {block_info}\n")
+ elif block_type == "autoclass":
+ if block_info is not None:
+ block = f"[[autodoc]] {block_info}\n"
+ options = parse_options(block_content)
+ if "special-members" in options:
+ special_members = options["special-members"].split(", ")
+ for special_member in special_members:
+ block += f" - {special_member}\n"
+ if "members" in options:
+ members = options["members"]
+ if len(members) == 0:
+ block += " - all\n"
+ else:
+ for member in members.split(", "):
+ block += f" - {member}\n"
+ new_lines.append(block)
+ elif block_type == "image":
+ options = parse_options(block_content)
+ target = options.pop("target", None)
+ if block_info is not None:
+ options["src"] = block_info
+ else:
+ if target is None:
+ raise ValueError("Image source not defined.")
+ options["src"] = target
+ # Adapt path
+ options["src"] = options["src"].replace("/imgs/", f"/docs/{package_name}/{version}/{language}/imgs/")
+ html_code = " ".join([f'{key}="{value}"' for key, value in options.items()])
+ new_lines.append(f"<img {html_code}/>\n")
+
+ else:
+ new_lines.append(f"{block_type},{block_info}\n{block_content.rstrip()}\n")
+
+ else:
+ new_lines.append(lines[idx])
+ idx += 1
+
+ return "\n".join(new_lines)
+
+
+# Re pattern that catches rst args blocks of the form `Parameters:`.
+_re_args = re.compile("^\s*(Args?|Arguments?|Attributes?|Params?|Parameters?):\s*$")
+# Re pattern that catches return blocks of the form `Return:`.
+_re_returns = re.compile("^\s*(Return|Yield|Raise)s?:\s*$")
+
+
+def split_return_line(line):
+ """
+ Split the return line with format `type: some doc`. Type may contain colons in the form of :obj: or :class:.
+ """
+ splits_on_colon = line.split(":")
+ idx = 1
+ while idx < len(splits_on_colon) and splits_on_colon[idx] in ["obj", "class"]:
+ idx += 2
+ if idx >= len(splits_on_colon):
+ if len(splits_on_colon) % 2 == 1 and re.search(r"`\w+`$", line.rstrip()):
+ return line, ""
+ return None, line
+ return ":".join(splits_on_colon[:idx]), ":".join(splits_on_colon[idx:])
+
+
+def split_raise_line(line):
+ """
+ Split the raise line with format `SomeError some doc`.
+ """
+ splits_on_colon = line.strip().split(" ")
+ error_type, doc = splits_on_colon[0], " ".join(splits_on_colon[1:])
+ if error_type and error_type[-1] == ":":
+ error_type = error_type[:-1]
+ return error_type, doc
+
+
+def split_arg_line(line):
+ """
+ Split the return line with format `type: some doc`. Type may contain colons in the form of :obj: or :class:.
+ """
+ splits_on_colon = line.split(":")
+ idx = 1
+ while idx < len(splits_on_colon) and splits_on_colon[idx] in ["obj", "class"]:
+ idx += 2
+ if idx >= len(splits_on_colon):
+ return line, ""
+ return ":".join(splits_on_colon[:idx]), ":".join(splits_on_colon[idx:])
+
+
+class InvalidRstDocstringError(ValueError):
+ pass
+
+
+_re_parameters = re.compile(r"<parameters>(((?!<parameters>).)*)</parameters>", re.DOTALL)
+_re_md_link = re.compile(r"\[(.+)\]\(.+\)", re.DOTALL)
+
+
+def parse_rst_docstring(docstring):
+ """
+ Parses a docstring written in rst, in particular the list of arguments and the return type.
+ """
+ lines = docstring.split("\n")
+ idx = 0
+ while idx < len(lines):
+ # Parameters section
+ if _re_args.search(lines[idx]) is not None:
+ # Title of the section.
+ lines[idx] = "<parameters>\n"
+ # Find the next nonempty line
+ idx += 1
+ while is_empty_line(lines[idx]):
+ idx += 1
+ # Grab the indent of the list of parameters, this block will stop when we unindent under it or we see the
+ # Returns or Raises block.
+ param_indent = find_indent(lines[idx])
+ while (
+ idx < len(lines) and find_indent(lines[idx]) == param_indent and _re_returns.search(lines[idx]) is None
+ ):
+ intro, doc = split_arg_line(lines[idx])
+ # Line starting with a > after indent indicate a "section title" in the parameters.
+ if intro.lstrip().startswith(">"):
+ lines[idx] = intro.lstrip()
+ else:
+ lines[idx] = re.sub(r"^\s*(\S+)(\s)", r"- **\1**\2", intro) + " --" + doc
+ idx += 1
+ while idx < len(lines) and (is_empty_line(lines[idx]) or find_indent(lines[idx]) > param_indent):
+ idx += 1
+ lines.insert(idx, "</parameters>\n")
+ idx += 1
+
+ # Returns section
+ elif _re_returns.search(lines[idx]) is not None:
+ # tag is either `return` or `yield`
+ tag = _re_returns.match(lines[idx]).group(1).lower()
+ # Title of the section.
+ lines[idx] = f"<{tag}s>\n"
+ # Find the next nonempty line
+ idx += 1
+ while is_empty_line(lines[idx]):
+ idx += 1
+
+ # Grab the indent of the return line, this block will stop when we unindent under it.
+ return_indent = find_indent(lines[idx])
+ raised_errors = []
+ # The line may contain the return type.
+ if tag in ["return", "yield"]:
+ return_type, return_description = split_return_line(lines[idx])
+ lines[idx] = return_description
+ idx += 1
+ while idx < len(lines) and (is_empty_line(lines[idx]) or find_indent(lines[idx]) >= return_indent):
+ idx += 1
+ else:
+ while idx < len(lines) and find_indent(lines[idx]) == return_indent:
+ return_type, return_description = split_raise_line(lines[idx])
+ raised_error = re.sub(r"^\s*`?([\w\.]*)`?$", r"``\1``", return_type)
+ lines[idx] = "- " + raised_error + " -- " + return_description
+ md_link = _re_md_link.match(raised_error)
+ if md_link:
+ raised_error = md_link[1]
+ raised_error = re.sub(r"^\s*`?([\w\.]*)`?$", r"``\1``", raised_error)
+ if raised_error not in raised_errors:
+ raised_errors.append(raised_error)
+ idx += 1
+ while idx < len(lines) and (is_empty_line(lines[idx]) or find_indent(lines[idx]) > return_indent):
+ idx += 1
+
+ lines.insert(idx, f"</{tag}s>\n")
+ idx += 1
+
+ # Return block finished, we insert the return type if one was specified
+ if tag in ["return", "yield"] and return_type is not None:
+ lines[idx - 1] += f"\n<{tag}type>{return_type}</{tag}type>\n"
+ elif len(raised_errors) > 0:
+ # raised errors
+ lines[idx - 1] += f"\n<raisederrors>{' or '.join(raised_errors)}</raisederrors>\n"
+
+ else:
+ idx += 1
+
+ result = "\n".join(lines)
+
+ # combine multiple <parameters> blocks into one block
+ if result.count("<parameters>") > 1:
+ parameters_blocks = _re_parameters.findall(result)
+ parameters_blocks = [pb[0].strip() for pb in parameters_blocks]
+ parameters_str = "\n".join(parameters_blocks)
+ result = _re_parameters.sub("", result)
+ result += f"\n<parameters>{parameters_str}</parameters>\n"
+
+ return result
+
+
+_re_list = re.compile("^\s*(-|\*|\d+\.)\s")
+_re_autodoc = re.compile("^\s*\[\[autodoc\]\]\s+(\S+)\s*$")
+
+
+def remove_indent(text):
+ """
+ Remove indents in text, except the one linked to lists (or sublists).
+ """
+ lines = text.split("\n")
+ # List of indents to remember for nested lists
+ current_indents = []
+ # List of new indents to remember for nested lists
+ new_indents = []
+ is_inside_code = False
+ code_indent = 0
+ for idx, line in enumerate(lines):
+ # Line is an item in a list.
+ if _re_list.search(line) is not None:
+ indent = find_indent(line)
+ # Is it a new list / new level of nestedness?
+ if len(current_indents) == 0 or indent > current_indents[-1]:
+ current_indents.append(indent)
+ new_indent = 0 if len(new_indents) == 0 else new_indents[-1]
+ lines[idx] = " " * new_indent + line[indent:]
+ new_indent += len(_re_list.search(line).groups()[0]) + 1
+ new_indents.append(new_indent)
+ # Otherwise it's an existing level of list (current one, or previous one)
+ else:
+ # Let's find the proper level of indentation
+ level = len(current_indents) - 1
+ while level >= 0 and current_indents[level] != indent:
+ level -= 1
+ current_indents = current_indents[: level + 1]
+ new_indents = new_indents[:level]
+ new_indent = 0 if len(new_indents) == 0 else new_indents[-1]
+ lines[idx] = " " * new_indent + line[indent:]
+ new_indent += len(_re_list.search(line).groups()[0]) + 1
+ new_indents.append(new_indent)
+
+ # Line is an autodoc, we keep the indent for the list just after if there is one.
+ elif _re_autodoc.search(line) is not None:
+ indent = find_indent(line)
+ current_indents = [indent]
+ new_indents = [4]
+ lines[idx] = line.strip()
+
+ # Deal with empty lines separately
+ elif is_empty_line(line):
+ lines[idx] = ""
+
+ # Code blocks
+ elif line.lstrip().startswith("```"):
+ is_inside_code = not is_inside_code
+ if is_inside_code:
+ code_indent = find_indent(line)
+ lines[idx] = line[code_indent:]
+ elif is_inside_code:
+ lines[idx] = line[code_indent:]
+
+ else:
+ indent = find_indent(line)
+ if len(current_indents) > 0 and indent > current_indents[-1]:
+ lines[idx] = " " * new_indents[-1] + line[indent:]
+ elif len(current_indents) > 0:
+ # Let's find the proper level of indentation
+ level = len(current_indents) - 1
+ while level >= 0 and current_indents[level] > indent:
+ level -= 1
+ current_indents = current_indents[: level + 1]
+ if level >= 0:
+ if current_indents[level] < indent:
+ new_indents = new_indents[: level + 1]
+ else:
+ new_indents = new_indents[:level]
+ new_indent = 0 if len(new_indents) == 0 else new_indents[-1]
+ lines[idx] = " " * new_indent + line[indent:]
+ new_indents.append(new_indent)
+ else:
+ new_indents = []
+ lines[idx] = line[indent:]
+ else:
+ lines[idx] = line[indent:]
+
+ return "\n".join(lines)
+
+
+def base_rst_to_mdx(text, page_info, unindent=True):
+ """
+ Convert a text from rst to mdx, with the base operations necessary for both docstrings and rst docs.
+ """
+ text = convert_rst_links(text, page_info)
+ text = convert_special_chars(text)
+ text = convert_rst_blocks(text, page_info)
+ # Convert * in lists to - to avoid the formatting conversion treat them as bold.
+ text = re.sub(r"^(\s*)\*(\s)", r"\1-\2", text, flags=re.MULTILINE)
+ text = convert_rst_formatting(text)
+ return remove_indent(text) if unindent else text
+
+
+def convert_rst_docstring_to_mdx(docstring, page_info):
+ """
+ Convert a docstring written in rst to mdx.
+ """
+ text = parse_rst_docstring(docstring)
+ return base_rst_to_mdx(text, page_info)
+
+
+def process_titles(lines):
+ """Converts rst titles to markdown titles."""
+ title_chars = """= - ` : ' " ~ ^ _ * + # < >""".split(" ")
+ title_levels = {}
+ new_lines = []
+ for line in lines:
+ if (
+ len(new_lines) > 0
+ and len(line) >= len(new_lines[-1])
+ and len(set(line)) == 1
+ and line[0] in title_chars
+ and line != "::"
+ ):
+ char = line[0]
+ level = title_levels.get(char, len(title_levels) + 1)
+ if level not in title_levels:
+ title_levels[char] = level
+ new_lines[-1] = f"{'#' * level} {new_lines[-1]}"
+ else:
+ new_lines.append(line)
+ return new_lines
+
+
+# Matches lines with a pattern of a table new line in rst.
+_re_ignore_line_table = re.compile("^(\+[\-\s]+)+\+\s*$")
+# Matches lines with a pattern of a table new line in rst, with a first column empty.
+_re_ignore_line_table1 = re.compile("^\|\s+(\+[\-\s]+)+\+\s*$")
+# Matches lines with a pattern of a first table line in rst.
+_re_sep_line_table = re.compile("^(\+[=\s]+)+\+\s*$")
+# Re pattern that catches anchors of the type .. reference:
+_re_anchor_section = re.compile(r"^\.\.\s+_(\S+):")
+
+
+def split_pt_tf_code_blocks(text):
+ """
+ Split PyTorch and TensorFlow specific block codes.
+ """
+ lines = text.split("\n")
+ new_lines = []
+ idx = 0
+ while idx < len(lines):
+ if lines[idx].startswith("```"):
+ code_lines = {"common": [lines[idx]], "pytorch": [], "tensorflow": []}
+ is_pytorch = False
+ is_tensorflow = False
+ idx += 1
+ while idx < len(lines) and lines[idx].strip() != "```":
+ if "## PYTORCH CODE" in lines[idx]:
+ is_pytorch = True
+ is_tensorflow = False
+ elif "## TENSORFLOW CODE" in lines[idx]:
+ is_tensorflow = True
+ is_pytorch = False
+ elif is_pytorch:
+ code_lines["pytorch"].append(lines[idx])
+ elif is_tensorflow:
+ code_lines["tensorflow"].append(lines[idx])
+ else:
+ code_lines["common"].append(lines[idx])
+ idx += 1
+ if len(code_lines["pytorch"]) > 0 or len(code_lines["tensorflow"]) > 0:
+ block_lines = ["<frameworkcontent>", "<pt>"]
+ block_lines.extend(code_lines["common"].copy() + code_lines["pytorch"])
+ block_lines.extend(["```", "</pt>", "<tf>"])
+ block_lines.extend(code_lines["common"].copy() + code_lines["tensorflow"])
+ block_lines.extend(["```", "</tf>", "</frameworkcontent>"])
+ new_lines.extend(block_lines)
+ else:
+ block_lines = code_lines["common"] + ["```"]
+ new_lines.extend(block_lines)
+ idx += 1
+ else:
+ new_lines.append(lines[idx])
+ idx += 1
+ return "\n".join(new_lines)
+
+
+def convert_rst_to_mdx(rst_text, page_info, add_imports=True):
+ """
+ Convert a document written in rst to mdx.
+ """
+ lines = rst_text.split("\n")
+ lines = process_titles(lines)
+ if add_imports:
+ new_lines = [
+ '<script lang="ts">',
+ ' import Tip from "$lib/Tip.svelte";',
+ ' import Youtube from "$lib/Youtube.svelte";',
+ ' import Docstring from "$lib/Docstring.svelte";',
+ ' import CodeBlock from "$lib/CodeBlock.svelte";',
+ ' import CodeBlockFw from "$lib/CodeBlockFw.svelte";',
+ ' import DocNotebookDropdown from "$lib/DocNotebookDropdown.svelte";',
+ ' import CourseFloatingBanner from "$lib/CourseFloatingBanner.svelte";',
+ ' import IconCopyLink from "$lib/IconCopyLink.svelte";',
+ ' import FrameworkContent from "$lib/FrameworkContent.svelte";',
+ ' import Markdown from "$lib/Markdown.svelte";',
+ ' import ExampleCodeBlock from "$lib/ExampleCodeBlock.svelte";',
+ ' import Added from "$lib/Added.svelte";',
+ ' import Changed from "$lib/Changed.svelte";',
+ ' import Deprecated from "$lib/Deprecated.svelte";',
+ ' import PipelineIcon from "$lib/PipelineIcon.svelte";',
+ ' import PipelineTag from "$lib/PipelineTag.svelte";',
+ " ",
+ ' export let fw: "pt" | "tf"',
+ "</script>",
+ "<svelte:head>",
+ '<meta name="hf:doc:metadata" content={JSON.stringify(metadata)} >',
+ "</svelte:head>",
+ "",
+ ]
+ else:
+ new_lines = []
+ for line in lines:
+ if _re_ignore_line_table.search(line) is not None:
+ continue
+ elif _re_ignore_line_table1.search(line) is not None:
+ continue
+ elif _re_sep_line_table.search(line) is not None:
+ line = line.replace("=", "-").replace("+", "|")
+ elif _re_anchor_section.search(line) is not None:
+ anchor_name = _re_anchor_section.search(line).groups()[0]
+ line = f"<a id='{anchor_name}'></a>"
+ new_lines.append(line)
+ text = "\n".join(new_lines)
+
+ return split_pt_tf_code_blocks(base_rst_to_mdx(text, page_info))
diff --git a/doc-build/third_party/hf-doc-builder/src/doc_builder/convert_to_notebook.py b/doc-build/third_party/hf-doc-builder/src/doc_builder/convert_to_notebook.py
new file mode 100644
index 00000000..59a4f1e3
--- /dev/null
+++ b/doc-build/third_party/hf-doc-builder/src/doc_builder/convert_to_notebook.py
@@ -0,0 +1,319 @@
+# coding=utf-8
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+from pathlib import Path
+
+import nbformat
+
+from .autodoc import resolve_links_in_text
+from .convert_md_to_mdx import clean_doctest_syntax
+from .convert_rst_to_mdx import is_empty_line
+from .utils import get_doc_config
+
+
+# Re pattern that matches inline math in MDX: \\(formula\\)
+_re_math_delimiter = re.compile(r"\\\\\((.*?)\\\\\)")
+# Re pattern that matches the copyright paragraph in an MDX file
+_re_copyright = re.compile("<!--\s*Copyright(.*?)-->", flags=re.DOTALL)
+# Re pattern that matches YouTube Svelte components and extract the id
+_re_youtube = re.compile(r'<Youtube id="([^"]+)"/>')
+# Re pattern matching header lines in Markdown
+_re_header = re.compile("^#+\s+\S+")
+# Re pattern matching Python code samples
+_re_python_code = re.compile("^```\s*(py|python)\s*$")
+# Re pattern matching markdown links
+_re_markdown_links = re.compile(r"\[([^\]]*)\]\(([^\)]*)\)")
+# Re pattern matching framework headers like <pytorch> or <tensorflow>
+_re_framework = re.compile("^\s*<([a-z]*)>\s*$")
+
+
+def expand_links(content, page_info):
+ """
+ Expand links relative to the documentation to full links to the hf.co website.
+ """
+ package_name = page_info["package_name"]
+ version = page_info.get("version", "main")
+ language = page_info.get("language", "en")
+ page = str(page_info["page"])
+
+ prefix = f"https://huggingface.co/docs/{package_name}/{version}/{language}"
+
+ def _replace_link(match):
+ description, link = match.groups()
+ if link.startswith("http") or link.startswith("#"):
+ return f"[{description}]({link})"
+ elif link.startswith("/docs/"):
+ return f"[{description}](https://huggingface.co{link})"
+ link = "/".join([prefix] + page.split("/")[:-1] + [link])
+ return f"[{description}]({link})"
+
+ return _re_markdown_links.sub(_replace_link, content)
+
+
+def clean_content(content, package=None, mapping=None, page_info=None):
+ """
+ Clean the content of the doc file to be pure Markdown.
+ """
+ # Remove copyright
+ content = _re_copyright.sub("", content)
+ # Remove [[open-in-colab]] marker
+ content = content.replace("[[open-in-colab]]\n", "")
+ # Replace our special syntax for math formula with the one expected in a notebook.
+ content = _re_math_delimiter.sub(r"$\1$", content)
+ # Resolve the doc links if possible
+ if package is not None and mapping is not None and page_info is not None:
+ content = resolve_links_in_text(content, package, mapping, page_info)
+ content = expand_links(content, page_info)
+
+ return content.strip()
+
+
+def split_frameworks(content):
+ """
+ Split a given doc content in three to extract the Mixed, PyTorch and TensorFlow content.
+ """
+ new_lines = {"mixed": [], "pt": [], "tf": []}
+
+ content = clean_doctest_syntax(content)
+ lines = content.split("\n")
+ idx = 0
+ while idx < len(lines):
+ if lines[idx].strip() == "<frameworkcontent>":
+ idx += 1
+ current_lines = []
+ current_framework = None
+ while idx < len(lines) and lines[idx].strip() != "</frameworkcontent>":
+ if _re_framework.search(lines[idx]) is not None:
+ current_framework = _re_framework.search(lines[idx]).groups()[0]
+ elif current_framework is not None and lines[idx].strip() == f"</{current_framework}>":
+ new_lines[current_framework].extend(current_lines)
+ new_lines["mixed"].extend(current_lines)
+ current_framework = None
+ current_lines = []
+ elif current_framework is not None:
+ current_lines.append(lines[idx])
+ idx += 1
+ idx += 1
+ else:
+ for key in new_lines.keys():
+ new_lines[key].append(lines[idx])
+ idx += 1
+ return ["\n".join(l) for l in new_lines.values()]
+
+
+def markdown_cell(content):
+ """
+ Create a markdown cell with a given content.
+ """
+ return nbformat.notebooknode.NotebookNode({"cell_type": "markdown", "source": content, "metadata": {}})
+
+
+def parse_input_output(code_lines):
+ """
+ Parse a code sample written in doctest syntax to extract input code and expected output.
+ """
+ current_lines = []
+ in_input = True
+ cells = []
+
+ for idx, line in enumerate(code_lines):
+ if is_empty_line(line):
+ current_lines.append(line)
+ elif not in_input and line.startswith(">>> "):
+ in_input = True
+ cells[-1] = (cells[-1][0], "\n".join(current_lines).strip())
+ current_lines = [line[4:]]
+ elif in_input and line[:4] not in [">>> ", "... "]:
+ in_input = False
+ cells.append(("\n".join(current_lines).strip(), None))
+ current_lines = [line]
+ else:
+ if line.startswith(">>> ") or line.startswith("... "):
+ current_lines.append(line[4:])
+ else:
+ current_lines.append(line)
+
+ if in_input:
+ cells.append(("\n".join(current_lines).strip(), None))
+ else:
+ cells[-1] = (cells[-1][0], "\n".join(current_lines).strip())
+
+ if len(cells) == 1 and len(cells[0][0]) == 0:
+ return [(cells[0][1], None)]
+
+ return cells
+
+
+def code_cell(code, output=None):
+ """
+ Create a code cell with some `code` and optionally, `output`.
+ """
+ if output is None or len(output) == 0:
+ outputs = []
+ else:
+ outputs = [
+ nbformat.notebooknode.NotebookNode(
+ {
+ "data": {"text/plain": output},
+ "execution_count": None,
+ "metadata": {},
+ "output_type": "execute_result",
+ }
+ )
+ ]
+ return nbformat.notebooknode.NotebookNode(
+ {"cell_type": "code", "execution_count": None, "source": code, "metadata": {}, "outputs": outputs}
+ )
+
+
+def youtube_cell(youtube_id):
+ """
+ Create a "YouTube" cell for a given ID. It's actually a code cell with input hidden (requires the hide_input
+ extension in regular notebooks and works out of the box on Colab) and the output being the widget with the video.
+
+ Widgets won't be shown by default in a regular notebook unless the user clicks Trust notebook.
+ """
+ html_code = f'<iframe width="560" height="315" src="https://www.youtube.com/embed/{youtube_id}?rel=0&controls=0&showinfo=0" frameborder="0" allowfullscreen></iframe>'
+ cell_dict = {
+ "cell_type": "code",
+ "metadata": {"cellView": "form", "hide_input": True},
+ "source": ["#@title\n", "from IPython.display import HTML\n", "\n", f"HTML('{html_code}')"],
+ "execution_count": None,
+ }
+ output_dict = {
+ "data": {"text/html": [html_code], "text/plain": ["<IPython.core.display.HTML object>"]},
+ "execution_count": None,
+ "metadata": {},
+ "output_type": "execute_result",
+ }
+ cell_dict["outputs"] = [nbformat.notebooknode.NotebookNode(output_dict)]
+ return nbformat.notebooknode.NotebookNode(cell_dict)
+
+
+def parse_doc_into_cells(content):
+ """
+ Split a documentation content into cells.
+ """
+ cells = []
+ doc_config = get_doc_config()
+ if doc_config is not None and hasattr(doc_config, "notebook_first_cells"):
+ for cell in doc_config.notebook_first_cells:
+ if cell["type"] == "markdown":
+ cells.append(markdown_cell(cell["content"].strip()))
+ elif cell["type"] == "code":
+ cells.append(code_cell(cell["content"].strip()))
+
+ current_lines = []
+ cell_type = "markdown"
+ # We keep track of whether we are in a general code block (not necessarily in a code cell) as a line with a comment
+ # would be detected as a header.
+ in_code = False
+
+ for line in content.split("\n"):
+ # Look if we've got a special line.
+ special_line = None
+ if _re_header.search(line) is not None and not in_code:
+ special_line = "header"
+ elif _re_python_code.search(line) is not None:
+ special_line = "begin_code"
+ elif line.rstrip() == "```" and cell_type == "code":
+ special_line = "end_code"
+ elif line.startswith("```"):
+ special_line = "other_code"
+ elif _re_youtube.search(line) is not None:
+ special_line = "youtube"
+
+ # Some of those special lines mean we have to process the current cell.
+ process_current_cell = False
+ if cell_type == "markdown":
+ process_current_cell = special_line in ["header", "begin_code", "youtube"]
+ elif cell_type == "code":
+ process_current_cell = special_line == "end_code"
+
+ # Add the current cell to the list
+ if process_current_cell:
+ if cell_type == "markdown":
+ content = "\n".join(current_lines).strip()
+ if len(content) > 0:
+ cells.append(markdown_cell(content))
+ elif cell_type == "code" and len(current_lines) > 0:
+ for code, output in parse_input_output(current_lines):
+ cells.append(code_cell(code, output=output))
+ current_lines = []
+
+ if special_line == "header":
+ # Header go on their separate Markdown cell, as it plays nicely with the collapsible headers extension.
+ cells.append(markdown_cell(line))
+ elif special_line == "begin_code":
+ cell_type = "code"
+ in_code = True
+ elif special_line == "end_code":
+ cell_type = "markdown"
+ in_code = False
+ elif special_line == "other_code":
+ current_lines.append(line)
+ in_code = not in_code
+ elif special_line == "youtube":
+ # YouTube cells are their own separate cell for proper showing
+ youtube_id = _re_youtube.search(line).groups()[0]
+ cells.append(youtube_cell(youtube_id))
+ else:
+ current_lines.append(line)
+
+ # Now that we're done, we just have to process the remainder.
+ if cell_type == "markdown":
+ content = "\n".join(current_lines).strip()
+ if len(content) > 0:
+ cells.append(markdown_cell(content))
+
+ return cells
+
+
+def create_notebook(cells):
+ """
+ Create a notebook object with `cells`.
+ """
+ return nbformat.notebooknode.NotebookNode({"cells": cells, "metadata": {}, "nbformat": 4, "nbformat_minor": 4})
+
+
+def generate_notebooks_from_file(file_name, output_dir, package=None, mapping=None, page_info=None):
+ """
+ Generate the notebooks for a given doc file.
+
+ Args:
+ file_name (`str` or `os.PathLike`): The doc file to convert.
+ output_dir (`str` or `os.PathLike`): Where to save the generated notebooks
+ package (`types.ModuleType`, *optional*):
+ The package in which to search objects for (needs to be passed to resolve doc links).
+ mapping (`Dict[str, str]`, *optional*):
+ The map from anchor names of objects to their page in the documentation (needs to be passed to resolve doc
+ links).
+ page_info (`Dict[str, str]`, *optional*):
+ Some information about the page (needs to be passed to resolve doc links).
+ """
+ output_dirs = [output_dir, os.path.join(output_dir, "pytorch"), os.path.join(output_dir, "tensorflow")]
+ output_name = Path(file_name).with_suffix(".ipynb").name
+ with open(file_name, "r", encoding="utf-8") as f:
+ content = f.read()
+
+ content = clean_content(content, package=package, mapping=mapping, page_info=page_info)
+
+ for folder, content in zip(output_dirs, split_frameworks(content)):
+ cells = parse_doc_into_cells(content)
+ notebook = create_notebook(cells)
+ os.makedirs(folder, exist_ok=True)
+ nbformat.write(notebook, os.path.join(folder, output_name), version=4)
diff --git a/doc-build/third_party/hf-doc-builder/src/doc_builder/external.py b/doc-build/third_party/hf-doc-builder/src/doc_builder/external.py
new file mode 100644
index 00000000..6d94a4e4
--- /dev/null
+++ b/doc-build/third_party/hf-doc-builder/src/doc_builder/external.py
@@ -0,0 +1,152 @@
+# coding=utf-8
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+import tempfile
+import zlib
+
+import git
+import requests
+
+
+HF_DOC_PREFIX = "https://huggingface.co/docs/"
+EXTERNAL_DOC_OBJECTS_CACHE = {}
+HUGGINFACE_LIBS = [
+ "accelerate",
+ "datasets",
+ "evaluate",
+ "huggingface_hub",
+ "optimum",
+ "tokenizers",
+ "transformers",
+]
+
+
+def post_process_objects_inv(object_data, doc_url):
+ """
+ Post-processes the data in sphinx-like format to get a dictionary object_name: link in doc.
+
+ Args:
+ object_data (`str`): The data in the `objects.inv` object (except the first 4 lines).
+ doc_url (`str`): The documentation url of the package.
+ """
+ links = {}
+ for line in object_data:
+ if len(line) == 0:
+ continue
+ name, typ, _, link = line.split(" ")[:4]
+ if typ in ["py:class", "py:function", "py:method"]:
+ link = link.replace("$", name)
+ links[name] = f"{doc_url}/{link}"
+ return links
+
+
+def get_stable_version(package_name, repo_owner="huggingface", repo_name=None):
+ """
+ Gets the version of the last release of a package.
+
+ Args:
+ package_name (`str`): The name of the package.
+ repo_owner (`str`): The owner of the GitHub repo.
+ repo_name (`str`, *optional*, defaults to `package_name`):
+ The name of the GitHub repo. If not provided, will be the same as the package name.
+ """
+ repo_name = repo_name if repo_name is not None else package_name
+ github_url = f"https://github.com/{repo_owner}/{repo_name}"
+ try:
+ # Get the version tags from the GitHub repo in decreasing order (that's what '-v:refname' means)
+ result = git.cmd.Git().ls_remote(github_url, sort="-v:refname", tags=True)
+ except git.GitCommandError:
+ return "main"
+
+ # One line per tag
+ for line in result.split("\n"):
+ # Lines returned are {sha}\trefs/tags/{tag}^{}, we grab the tag
+ candidate = line.split("/")[-1].replace("^", "").replace("{}", "")
+ # Some tags are not versions (looking at your VERSION and delete tags Datasets)
+ if re.search(r"v?\d+\.\d+\.\d+", candidate):
+ # Add the v is missing (looking at you Datasets :-p)
+ return candidate if candidate.startswith("v") else f"v{candidate}"
+
+ return "main"
+
+
+def get_objects_map(package_name, version="main", language="en", repo_owner="huggingface", repo_name=None):
+ """
+ Downloads the `objects.inv` for a package and post-processes it to get a nice dictionary.
+
+ Args:
+ package_name (`str`): The name of the external package.
+ version (`str`, *optional*, defaults to `"main"`): The version of the package for which documentation is built.
+ language (`str`, *optional*, defaults to `"en"`): The language of the documentation being built.
+ repo_owner (`str`, *optional*, defaults to `"huggingface"`): The owner of the GitHub repo.
+ repo_name (`str`, *optional*, defaults to `package_name`):
+ The name of the GitHub repo. If not provided, it will be the same as the package name.
+ """
+ repo_name = repo_name if repo_name is not None else package_name
+ # We link to main in `package_name` from the main doc (or PR docs) but to the last stable release otherwise.
+ if version in ["main", "master"] or version.startswith("pr_"):
+ package_version = "main"
+ else:
+ package_version = get_stable_version(package_name, repo_owner, repo_name)
+
+ doc_url = f"{HF_DOC_PREFIX}{package_name}/{package_version}/{language}"
+ url = f"{doc_url}/objects.inv"
+ try:
+ request = requests.get(url, stream=True)
+ request.raise_for_status()
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ tmp_file = os.path.join(tmp_dir, "objects.inv")
+ with open(tmp_file, "ab") as writer:
+ for chunk in request.iter_content(chunk_size=1024):
+ if chunk: # filter out keep-alive new chunks
+ writer.write(chunk)
+
+ with open(tmp_file, "rb") as reader:
+ object_lines = reader.readlines()[4:]
+ object_data = zlib.decompress(b"".join(object_lines)).decode().split("\n")
+ return post_process_objects_inv(object_data, doc_url)
+ except requests.HTTPError:
+ return {}
+
+
+def get_external_object_link(object_name, page_info):
+ if object_name.startswith("~"):
+ object_name = object_name[1:]
+ link_name = object_name.split(".")[-1]
+ else:
+ link_name = object_name
+
+ version = page_info.get("version", "main")
+ language = page_info.get("language", "en")
+ if language != "en":
+ # No resolving for other languages then English as we don't translate API doc pages/docstrings for now.
+ return f"`{link_name}`"
+
+ package_name = object_name.split(".")[0]
+ if package_name not in HUGGINFACE_LIBS:
+ # No resolving for non-HF libs for now.
+ return f"`{link_name}`"
+
+ if package_name not in EXTERNAL_DOC_OBJECTS_CACHE:
+ EXTERNAL_DOC_OBJECTS_CACHE[package_name] = get_objects_map(package_name, version=version, language=language)
+ object_url = EXTERNAL_DOC_OBJECTS_CACHE[package_name].get(object_name, None)
+
+ if object_url is None:
+ # Object not found in the lib
+ return f"`{link_name}`"
+ else:
+ return f"[{link_name}]({object_url})"
diff --git a/doc-build/third_party/hf-doc-builder/src/doc_builder/frontmatter_node.py b/doc-build/third_party/hf-doc-builder/src/doc_builder/frontmatter_node.py
new file mode 100644
index 00000000..0f699470
--- /dev/null
+++ b/doc-build/third_party/hf-doc-builder/src/doc_builder/frontmatter_node.py
@@ -0,0 +1,59 @@
+# coding=utf-8
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import yaml
+
+
+class FrontmatterNode:
+ """
+ A helper class that is used in generating frontmatter for mdx files.
+
+ This class is a typical graph node class, which allows it to model
+ markdown header hierarchy (i.e. h2 belongs to h1 above it).
+ """
+
+ def __init__(self, title, local):
+ self.title = title
+ self.local = local
+ self.children = []
+
+ def add_child(self, child, header_level):
+ parent = self
+ nested_level = header_level - 2
+ while nested_level:
+ if not parent.children:
+ parent.children.append(FrontmatterNode(None, None))
+ parent = parent.children[-1]
+ nested_level -= 1
+ parent.children.append(child)
+
+ def get_frontmatter(self):
+ self._remove_null_nodes()
+ frontmatter_yaml = yaml.dump(self._dictionarify(), allow_unicode=True).replace("\\U0001F917", "🤗")
+ return f"---\n{frontmatter_yaml}---\n"
+
+ def _remove_null_nodes(self):
+ if len(self.children) == 1 and self.children[0].title is None:
+ child_children = self.children[0].children
+ self.children = child_children
+ for section in self.children:
+ section._remove_null_nodes()
+
+ def _dictionarify(self):
+ if not self.children:
+ return {"title": self.title, "local": self.local}
+ children = [section._dictionarify() for section in self.children]
+ return {"title": self.title, "local": self.local, "sections": children}
diff --git a/doc-build/third_party/hf-doc-builder/src/doc_builder/style_doc.py b/doc-build/third_party/hf-doc-builder/src/doc_builder/style_doc.py
new file mode 100644
index 00000000..908eb2da
--- /dev/null
+++ b/doc-build/third_party/hf-doc-builder/src/doc_builder/style_doc.py
@@ -0,0 +1,548 @@
+# coding=utf-8
+# Copyright 2022 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Style utils for the markdown files and the docstrings."""
+
+import os
+import re
+import warnings
+
+import black
+
+from .convert_rst_to_mdx import _re_args, _re_returns, find_indent, is_empty_line
+from .utils import get_doc_config
+
+
+# Regexes
+# Re pattern that catches list introduction (with potential indent)
+_re_list = re.compile(r"^(\s*-\s+|\s*\*\s+|\s*\d+\.\s+)")
+# Re pattern that catches code block introduction (with potentinal indent)
+_re_code = re.compile(r"^(\s*)```(.*)$")
+# Matches the special tag to ignore some paragraphs.
+_re_docstyle_ignore = re.compile(r"#\s*docstyle-ignore")
+# Re pattern that matches <Tip>, </Tip> and <Tip warning={true}> blocks.
+_re_tip = re.compile("^\s*</?Tip(>|\s+warning={true}>)\s*$")
+
+DOCTEST_PROMPTS = [">>>", "..."]
+
+
+def get_black_avoid_patterns():
+ patterns = {"===PT-TF-SPLIT===": "### PT-TF-SPLIT"}
+ doc_config = get_doc_config()
+ if doc_config is not None and hasattr(doc_config, "black_avoid_patterns"):
+ patterns.update(doc_config.black_avoid_patterns)
+ return patterns
+
+
+def parse_code_example(code_lines):
+ """
+ Parses a code example
+
+ Args:
+ code_lines (`List[str]`): The code lines to parse.
+ max_len (`int`): The maximum length per line.
+
+ Returns:
+ (List[`str`], List[`str`]): The list of code samples and the list of outputs.
+ """
+ has_doctest = code_lines[0][:3] in DOCTEST_PROMPTS
+
+ code_samples = []
+ outputs = []
+ in_code = True
+ current_bit = []
+
+ for line in code_lines:
+ if in_code and has_doctest and not is_empty_line(line) and line[:3] not in DOCTEST_PROMPTS:
+ code_sample = "\n".join(current_bit)
+ code_samples.append(code_sample.strip())
+ in_code = False
+ current_bit = []
+ elif not in_code and line[:3] in DOCTEST_PROMPTS:
+ output = "\n".join(current_bit)
+ outputs.append(output.strip())
+ in_code = True
+ current_bit = []
+
+ # Add the line without doctest prompt
+ if line[:3] in DOCTEST_PROMPTS:
+ line = line[4:]
+ current_bit.append(line)
+
+ # Add last sample
+ if in_code:
+ code_sample = "\n".join(current_bit)
+ code_samples.append(code_sample.strip())
+ else:
+ output = "\n".join(current_bit)
+ outputs.append(output.strip())
+
+ return code_samples, outputs
+
+
+def format_code_example(code: str, max_len: int, in_docstring: bool = False):
+ """
+ Format a code example using black. Will take into account the doctest syntax as well as any initial indentation in
+ the code provided.
+
+ Args:
+ code (`str`): The code example to format.
+ max_len (`int`): The maximum length per line.
+ in_docstring (`bool`, *optional*, defaults to `False`): Whether or not the code example is inside a docstring.
+
+ Returns:
+ `str`: The formatted code.
+ """
+ code_lines = code.split("\n")
+
+ # Find initial indent
+ idx = 0
+ while idx < len(code_lines) and is_empty_line(code_lines[idx]):
+ idx += 1
+ if idx >= len(code_lines):
+ return "", ""
+ indent = find_indent(code_lines[idx])
+
+ # Remove the initial indent for now, we will had it back after styling.
+ # Note that l[indent:] works for empty lines
+ code_lines = [l[indent:] for l in code_lines[idx:]]
+ has_doctest = code_lines[0][:3] in DOCTEST_PROMPTS
+
+ code_samples, outputs = parse_code_example(code_lines)
+
+ # Let's blackify the code! We put everything in one big text to go faster.
+ delimiter = "\n\n### New code sample ###\n"
+ full_code = delimiter.join(code_samples)
+ line_length = max_len - indent
+ if has_doctest:
+ line_length -= 4
+
+ black_avoid_patterns = get_black_avoid_patterns()
+ for k, v in black_avoid_patterns.items():
+ full_code = full_code.replace(k, v)
+ try:
+ mode = black.Mode(target_versions={black.TargetVersion.PY37}, line_length=line_length)
+ formatted_code = black.format_str(full_code, mode=mode)
+ error = ""
+ except Exception as e:
+ formatted_code = full_code
+ error = f"Code sample:\n{full_code}\n\nError message:\n{e}"
+
+ # Let's get back the formatted code samples
+ for k, v in black_avoid_patterns.items():
+ formatted_code = formatted_code.replace(v, k)
+ # Triple quotes will mess docstrings.
+ if in_docstring:
+ formatted_code = formatted_code.replace('"""', "'''")
+
+ code_samples = formatted_code.split(delimiter)
+ # We can have one output less than code samples
+ if len(outputs) == len(code_samples) - 1:
+ outputs.append("")
+
+ formatted_lines = []
+ for code_sample, output in zip(code_samples, outputs):
+ # black may have added some new lines, we remove them
+ code_sample = code_sample.strip()
+ in_triple_quotes = False
+ in_decorator = False
+ for line in code_sample.strip().split("\n"):
+ if has_doctest and not is_empty_line(line):
+ prefix = (
+ "... "
+ if line.startswith(" ") or line[0] in [")", "]", "}"] or in_triple_quotes or in_decorator
+ else ">>> "
+ )
+ else:
+ prefix = ""
+ indent_str = "" if is_empty_line(line) else (" " * indent)
+ formatted_lines.append(indent_str + prefix + line)
+
+ if '"""' in line:
+ in_triple_quotes = not in_triple_quotes
+ if line.startswith(" "):
+ in_decorator = False
+ if line.startswith("@"):
+ in_decorator = True
+
+ formatted_lines.extend([" " * indent + line for line in output.split("\n")])
+ if not output.endswith("===PT-TF-SPLIT==="):
+ formatted_lines.append("")
+
+ result = "\n".join(formatted_lines)
+ return result.rstrip(), error
+
+
+def format_text(text, max_len, prefix="", min_indent=None):
+ """
+ Format a text in the biggest lines possible with the constraint of a maximum length and an indentation.
+
+ Args:
+ text (`str`): The text to format
+ max_len (`int`): The maximum length per line to use
+ prefix (`str`, *optional*, defaults to `""`): A prefix that will be added to the text.
+ The prefix doesn't count toward the indent (like a - introducing a list).
+ min_indent (`int`, *optional*): The minimum indent of the text.
+ If not set, will default to the length of the `prefix`.
+
+ Returns:
+ `str`: The formatted text.
+ """
+ text = re.sub(r"\s+", " ", text).strip()
+ if min_indent is not None:
+ if len(prefix) < min_indent:
+ prefix = " " * (min_indent - len(prefix)) + prefix
+
+ indent = " " * len(prefix)
+ new_lines = []
+ words = text.split(" ")
+ current_line = f"{prefix}{words[0]}"
+ for word in words[1:]:
+ try_line = f"{current_line} {word}"
+ if len(try_line) > max_len:
+ new_lines.append(current_line)
+ current_line = f"{indent}{word}"
+ else:
+ current_line = try_line
+ new_lines.append(current_line)
+ return "\n".join(new_lines)
+
+
+def split_line_on_first_colon(line):
+ splits = line.split(":")
+ return splits[0], ":".join(splits[1:])
+
+
+def style_docstring(docstring, max_len):
+ """
+ Style a docstring by making sure there is no useless whitespace and the maximum horizontal space is used.
+
+ Args:
+ docstring (`str`): The docstring to style.
+ max_len (`int`): The maximum length of each line.
+
+ Returns:
+ `str`: The styled docstring
+ """
+ if is_empty_line(docstring):
+ return docstring
+
+ lines = docstring.split("\n")
+ new_lines = []
+
+ # Initialization
+ current_paragraph = None
+ current_indent = -1
+ in_code = False
+ param_indent = -1
+ prefix = ""
+ black_errors = []
+
+ # Special case for docstrings that begin with continuation of Args with no Args block.
+ idx = 0
+ while idx < len(lines) and is_empty_line(lines[idx]):
+ idx += 1
+ if (
+ len(lines[idx]) > 1
+ and lines[idx].rstrip().endswith(":")
+ and find_indent(lines[idx + 1]) > find_indent(lines[idx])
+ ):
+ param_indent = find_indent(lines[idx])
+
+ idx = 0
+ while idx < len(lines):
+ line = lines[idx]
+ # Doing all re searches once for the ones we need to repeat.
+ list_search = _re_list.search(line)
+ code_search = _re_code.search(line)
+ args_search = _re_args.search(line)
+ tip_search = _re_tip.search(line)
+
+ # Are we starting a new paragraph?
+ # New indentation or new line:
+ new_paragraph = find_indent(line) != current_indent or is_empty_line(line)
+ # List item
+ new_paragraph = new_paragraph or list_search is not None
+ # Code block beginning
+ new_paragraph = new_paragraph or code_search is not None
+ # Beginning/end of tip
+ new_paragraph = new_paragraph or tip_search is not None
+ # Beginning of Args
+ new_paragraph = new_paragraph or args_search is not None
+
+ # In this case, we treat the current paragraph
+ if not in_code and new_paragraph and current_paragraph is not None and len(current_paragraph) > 0:
+ paragraph = " ".join(current_paragraph)
+ new_lines.append(format_text(paragraph, max_len, prefix=prefix, min_indent=current_indent))
+ # A blank line may be missing before the start of an argument block
+ if args_search is not None and not is_empty_line(current_paragraph[-1]):
+ new_lines.append("")
+ current_paragraph = None
+
+ if code_search is not None:
+ if not in_code:
+ current_paragraph = []
+ current_indent = len(code_search.groups()[0])
+ current_code = code_search.groups()[1]
+ prefix = ""
+ if current_indent < param_indent:
+ param_indent = -1
+ else:
+ current_indent = -1
+ code = "\n".join(current_paragraph)
+ if current_code in ["py", "python"]:
+ formatted_code, error = format_code_example(code, max_len, in_docstring=True)
+ new_lines.append(formatted_code)
+ if len(error) > 0:
+ black_errors.append(error)
+ else:
+ new_lines.append(code)
+ current_paragraph = None
+ new_lines.append(line)
+ in_code = not in_code
+
+ elif in_code:
+ current_paragraph.append(line)
+ elif is_empty_line(line):
+ current_paragraph = None
+ current_indent = -1
+ prefix = ""
+ new_lines.append(line)
+ elif list_search is not None:
+ prefix = list_search.groups()[0]
+ current_indent = len(prefix)
+ current_paragraph = [line[current_indent:]]
+ elif args_search:
+ new_lines.append(line)
+ idx += 1
+ while idx < len(lines) and is_empty_line(lines[idx]):
+ idx += 1
+ if idx < len(lines):
+ param_indent = find_indent(lines[idx])
+ # We still need to treat that line
+ idx -= 1
+ elif tip_search:
+ # Add a new line before if not present
+ if not is_empty_line(new_lines[-1]):
+ new_lines.append("")
+ new_lines.append(line)
+ # Add a new line after if not present
+ if idx < len(lines) - 1 and not is_empty_line(lines[idx + 1]):
+ new_lines.append("")
+ elif current_paragraph is None or find_indent(line) != current_indent:
+ indent = find_indent(line)
+ # Special behavior for parameters intros.
+ if indent == param_indent:
+ # Special rules for some docstring where the Returns blocks has the same indent as the parameters.
+ if _re_returns.search(line) is not None:
+ param_indent = -1
+ new_lines.append(line)
+ elif len(line) < max_len:
+ new_lines.append(line)
+ else:
+ intro, description = split_line_on_first_colon(line)
+ new_lines.append(intro + ":")
+ if len(description) != 0:
+ if find_indent(lines[idx + 1]) > indent:
+ current_indent = find_indent(lines[idx + 1])
+ else:
+ current_indent = indent + 4
+ current_paragraph = [description.strip()]
+ prefix = ""
+ else:
+ # Check if we have exited the parameter block
+ if indent < param_indent:
+ param_indent = -1
+
+ current_paragraph = [line.strip()]
+ current_indent = find_indent(line)
+ prefix = ""
+ elif current_paragraph is not None:
+ current_paragraph.append(line.lstrip())
+
+ idx += 1
+
+ if current_paragraph is not None and len(current_paragraph) > 0:
+ paragraph = " ".join(current_paragraph)
+ new_lines.append(format_text(paragraph, max_len, prefix=prefix, min_indent=current_indent))
+
+ return "\n".join(new_lines), "\n\n".join(black_errors)
+
+
+def style_docstrings_in_code(code, max_len=119):
+ """
+ Style all docstrings in some code.
+
+ Args:
+ code (`str`): The code in which we want to style the docstrings.
+ max_len (`int`): The maximum number of characters per line.
+
+ Returns:
+ `Tuple[str, str]`: A tuple with the clean code and the black errors (if any)
+ """
+ # fmt: off
+ splits = code.split('\"\"\"')
+ splits = [
+ (s if i % 2 == 0 or _re_docstyle_ignore.search(splits[i - 1]) is not None else style_docstring(s, max_len=max_len))
+ for i, s in enumerate(splits)
+ ]
+ black_errors = "\n\n".join([s[1] for s in splits if isinstance(s, tuple) and len(s[1]) > 0])
+ splits = [s[0] if isinstance(s, tuple) else s for s in splits]
+ clean_code = '\"\"\"'.join(splits)
+ # fmt: on
+
+ return clean_code, black_errors
+
+
+def style_file_docstrings(code_file, max_len=119, check_only=False):
+ """
+ Style all docstrings in a given file.
+
+ Args:
+ code_file (`str` or `os.PathLike`): The file in which we want to style the docstring.
+ max_len (`int`): The maximum number of characters per line.
+ check_only (`bool`, *optional*, defaults to `False`):
+ Whether to restyle file or just check if they should be restyled.
+
+ Returns:
+ `bool`: Whether or not the file was or should be restyled.
+ """
+ with open(code_file, "r", encoding="utf-8", newline="\n") as f:
+ code = f.read()
+
+ clean_code, black_errors = style_docstrings_in_code(code, max_len=max_len)
+
+ diff = clean_code != code
+ if not check_only and diff:
+ print(f"Overwriting content of {code_file}.")
+ with open(code_file, "w", encoding="utf-8", newline="\n") as f:
+ f.write(clean_code)
+
+ return diff, black_errors
+
+
+def style_mdx_file(mdx_file, max_len=119, check_only=False):
+ """
+ Style a MDX file by formatting all Python code samples.
+
+ Args:
+ mdx_file (`str` or `os.PathLike`): The file in which we want to style the examples.
+ max_len (`int`): The maximum number of characters per line.
+ check_only (`bool`, *optional*, defaults to `False`):
+ Whether to restyle file or just check if they should be restyled.
+
+ Returns:
+ `bool`: Whether or not the file was or should be restyled.
+ """
+ with open(mdx_file, "r", encoding="utf-8", newline="\n") as f:
+ content = f.read()
+
+ lines = content.split("\n")
+ current_code = []
+ current_language = ""
+ in_code = False
+ new_lines = []
+ black_errors = []
+
+ for line in lines:
+ if _re_code.search(line) is not None:
+ in_code = not in_code
+ if in_code:
+ current_language = _re_code.search(line).groups()[1]
+ current_code = []
+ else:
+ code = "\n".join(current_code)
+ if current_language in ["py", "python"]:
+ code, error = format_code_example(code, max_len)
+ if len(error) > 0:
+ black_errors.append(error)
+ new_lines.append(code)
+
+ new_lines.append(line)
+ elif in_code:
+ current_code.append(line)
+ else:
+ new_lines.append(line)
+
+ if in_code:
+ raise ValueError(f"There was a problem when styling {mdx_file}. A code block is opened without being closed.")
+
+ clean_content = "\n".join(new_lines)
+ diff = clean_content != content
+ if not check_only and diff:
+ print(f"Overwriting content of {mdx_file}.")
+ with open(mdx_file, "w", encoding="utf-8", newline="\n") as f:
+ f.write(clean_content)
+
+ return diff, "\n\n".join(black_errors)
+
+
+def style_doc_files(*files, max_len=119, check_only=False):
+ """
+ Applies doc styling or checks everything is correct in a list of files.
+
+ Args:
+ files (several `str` or `os.PathLike`): The files to treat.
+ max_len (`int`): The maximum number of characters per line.
+ check_only (`bool`, *optional*, defaults to `False`):
+ Whether to restyle file or just check if they should be restyled.
+
+ Returns:
+ List[`str`]: The list of files changed or that should be restyled.
+ """
+ changed = []
+ black_errors = []
+ for file in files:
+ # Treat folders
+ if os.path.isdir(file):
+ files = [os.path.join(file, f) for f in os.listdir(file)]
+ files = [f for f in files if os.path.isdir(f) or f.endswith(".mdx") or f.endswith(".py")]
+ changed += style_doc_files(*files, max_len=max_len, check_only=check_only)
+ # Treat mdx
+ elif file.endswith(".mdx"):
+ try:
+ diff, black_error = style_mdx_file(file, max_len=max_len, check_only=check_only)
+ if diff:
+ changed.append(file)
+ if len(black_error) > 0:
+ black_errors.append(
+ f"There was a problem while formatting an example in {file} with black:\m{black_error}"
+ )
+ except Exception:
+ print(f"There is a problem in {file}.")
+ raise
+ # Treat python files
+ elif file.endswith(".py"):
+ try:
+ diff, black_error = style_file_docstrings(file, max_len=max_len, check_only=check_only)
+ if diff:
+ changed.append(file)
+ if len(black_error) > 0:
+ black_errors.append(
+ f"There was a problem while formatting an example in {file} with black:\m{black_error}"
+ )
+ except Exception:
+ print(f"There is a problem in {file}.")
+ raise
+ else:
+ warnings.warn(f"Ignoring {file} because it's not a py or an mdx file or a folder.")
+ if len(black_errors) > 0:
+ black_message = "\n\n".join(black_errors)
+ raise ValueError(
+ "Some code examples can't be interpreted by black, which means they aren't regular python:\n\n"
+ + black_message
+ + "\n\nMake sure to fix the corresponding docstring or doc file, or remove the py/python after ``` if it "
+ + "was not supposed to be a Python code sample."
+ )
+ return changed
diff --git a/doc-build/third_party/hf-doc-builder/src/doc_builder/utils.py b/doc-build/third_party/hf-doc-builder/src/doc_builder/utils.py
new file mode 100644
index 00000000..ffd9faeb
--- /dev/null
+++ b/doc-build/third_party/hf-doc-builder/src/doc_builder/utils.py
@@ -0,0 +1,182 @@
+# coding=utf-8
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib.machinery
+import importlib.util
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+import yaml
+from packaging import version as package_version
+
+
+hf_cache_home = os.path.expanduser(
+ os.getenv("HF_HOME", os.path.join(os.getenv("XDG_CACHE_HOME", "~/.cache"), "huggingface"))
+)
+default_cache_path = os.path.join(hf_cache_home, "doc_builder")
+DOC_BUILDER_CACHE = os.getenv("DOC_BUILDER_CACHE", default_cache_path)
+
+
+def get_default_branch_name(repo_folder):
+ config = get_doc_config()
+ if config is not None and hasattr(config, "default_branch_name"):
+ print(config.default_branch_name)
+ return config.default_branch_name
+ try:
+ p = subprocess.run(
+ "git symbolic-ref refs/remotes/origin/HEAD".split(),
+ stderr=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ check=True,
+ encoding="utf-8",
+ cwd=repo_folder,
+ )
+ branch = p.stdout.strip().split("/")[-1]
+ return branch
+ except Exception:
+ # Just in case git is not installed, we need a default
+ return "main"
+
+
+def update_versions_file(build_path, version, doc_folder):
+ """
+ Insert new version into _versions.yml file of the library
+ Assumes that _versions.yml exists and has its first entry as main version
+ """
+ main_branch = get_default_branch_name(doc_folder)
+ if version == main_branch:
+ return
+ with open(os.path.join(build_path, "_versions.yml"), "r") as versions_file:
+ versions = yaml.load(versions_file, yaml.FullLoader)
+
+ if versions[0]["version"] != main_branch:
+ raise ValueError(f"{build_path}/_versions.yml does not contain a {main_branch} version")
+
+ main_version, sem_versions = versions[0], versions[1:]
+ new_version = {"version": version}
+ did_insert = False
+ for i, value in enumerate(sem_versions):
+ if package_version.parse(new_version["version"]) == package_version.parse(value["version"]):
+ # Nothing to do, the version is here already.
+ return
+ elif package_version.parse(new_version["version"]) > package_version.parse(value["version"]):
+ sem_versions.insert(i, new_version)
+ did_insert = True
+ break
+ if not did_insert:
+ sem_versions.append(new_version)
+
+ with open(os.path.join(build_path, "_versions.yml"), "w") as versions_file:
+ versions_updated = [main_version] + sem_versions
+ yaml.dump(versions_updated, versions_file)
+
+
+doc_config = None
+
+
+def read_doc_config(doc_folder):
+ """
+ Execute the `_config.py` file inside the doc source directory and executes it as a Python module.
+ """
+ global doc_config
+
+ if os.path.isfile(os.path.join(doc_folder, "_config.py")):
+ loader = importlib.machinery.SourceFileLoader("doc_config", os.path.join(doc_folder, "_config.py"))
+ spec = importlib.util.spec_from_loader("doc_config", loader)
+ doc_config = importlib.util.module_from_spec(spec)
+ loader.exec_module(doc_config)
+
+
+def get_doc_config():
+ """
+ Returns the `doc_config` if it has been loaded.
+ """
+ return doc_config
+
+
+def is_watchdog_available():
+ """
+ Checks if soft dependency `watchdog` exists.
+ """
+ return importlib.util.find_spec("watchdog") is not None
+
+
+def is_doc_builder_repo(path):
+ """
+ Detects whether a folder is the `doc_builder` or not.
+ """
+ setup_file = Path(path) / "setup.py"
+ if not setup_file.exists():
+ return False
+ with open(os.path.join(path, "setup.py")) as f:
+ first_line = f.readline()
+ return first_line == "# Doc-builder package setup.\n"
+
+
+def locate_kit_folder():
+ """
+ Returns the location of the `kit` folder of `doc-builder`.
+
+ Will clone the doc-builder repo and cache it, if it's not found.
+ """
+ # First try: let's search where the module is.
+ repo_root = Path(__file__).parent.parent.parent
+ kit_folder = repo_root / "kit"
+ if kit_folder.is_dir():
+ return kit_folder
+
+ # Second try, maybe we are inside the doc-builder repo
+ current_dir = Path.cwd()
+ while current_dir.parent != current_dir and not (current_dir / ".git").is_dir():
+ current_dir = current_dir.parent
+ kit_folder = current_dir / "kit"
+ if kit_folder.is_dir() and is_doc_builder_repo(current_dir):
+ return kit_folder
+
+ # Otherwise, let's clone the repo and cache it.
+ return Path(get_cached_repo()) / "kit"
+
+
+def get_cached_repo():
+ """
+ Clone and cache the `doc-builder` repo.
+ """
+ os.makedirs(DOC_BUILDER_CACHE, exist_ok=True)
+ cache_repo_path = Path(DOC_BUILDER_CACHE) / "doc-builder-repo"
+ if not cache_repo_path.is_dir():
+ print(
+ "To build the HTML doc, we need the kit subfolder of the `doc-builder` repo. Cloning it and caching at "
+ f"{cache_repo_path}."
+ )
+ _ = subprocess.run(
+ "git clone https://github.com/huggingface/doc-builder.git".split(),
+ stderr=subprocess.PIPE,
+ check=True,
+ encoding="utf-8",
+ cwd=DOC_BUILDER_CACHE,
+ )
+ shutil.move(Path(DOC_BUILDER_CACHE) / "doc-builder", cache_repo_path)
+ else:
+ _ = subprocess.run(
+ ["git", "pull"],
+ stderr=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ check=True,
+ encoding="utf-8",
+ cwd=cache_repo_path,
+ )
+ return cache_repo_path
diff --git a/doc-build/third_party/hf-doc-builder/tests/data/convert_literalinclude_dummy.txt b/doc-build/third_party/hf-doc-builder/tests/data/convert_literalinclude_dummy.txt
new file mode 100644
index 00000000..808b13b5
--- /dev/null
+++ b/doc-build/third_party/hf-doc-builder/tests/data/convert_literalinclude_dummy.txt
@@ -0,0 +1,12 @@
+# START python_import_answer
+import scipy as sp
+# END python_import_answer
+
+# START python_import
+import numpy as np
+import pandas as pd
+# END python_import
+
+# START node_import
+import fs
+# END node_import"""
\ No newline at end of file
diff --git a/doc-build/third_party/hf-doc-builder/tests/test_autodoc.py b/doc-build/third_party/hf-doc-builder/tests/test_autodoc.py
new file mode 100644
index 00000000..260f3bcd
--- /dev/null
+++ b/doc-build/third_party/hf-doc-builder/tests/test_autodoc.py
@@ -0,0 +1,668 @@
+# coding=utf-8
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import inspect
+import unittest
+from dataclasses import dataclass
+from typing import List, Optional, Union
+
+import timm
+import transformers
+from doc_builder.autodoc import (
+ autodoc,
+ document_object,
+ find_documented_methods,
+ find_object_in_package,
+ format_signature,
+ get_shortest_path,
+ get_signature_component,
+ get_source_link,
+ get_type_name,
+ hashlink_example_codeblock,
+ is_dataclass_autodoc,
+ remove_example_tags,
+ resolve_links_in_text,
+)
+from transformers import BertModel, BertTokenizer, BertTokenizerFast
+from transformers.utils import PushToHubMixin
+
+
+# This is dynamic since the Transformers/timm libraries are not frozen.
+TEST_LINE_NUMBER = inspect.getsourcelines(transformers.utils.ModelOutput)[1]
+TEST_LINE_NUMBER2 = inspect.getsourcelines(transformers.pipeline)[1]
+TEST_LINE_NUMBER_TIMM = inspect.getsourcelines(timm.create_model)[1]
+
+TEST_DOCSTRING = """Constructs a BERTweet tokenizer, using Byte-Pair-Encoding.
+
+This tokenizer inherits from [`~transformers.PreTrainedTokenizer`] which contains most of the main methods.
+Users should refer to this superclass for more information regarding those methods.
+
+<parameters>
+
+- **vocab_file** (`str`) --
+ Path to the vocabulary file.
+- **merges_file** (`str`) --
+ Path to the merges file.
+- **normalization** (`bool`, _optional_, defaults to `False`) --
+ Whether or not to apply a normalization preprocess.
+
+<Tip>
+
+When building a sequence using special tokens, this is not the token that is used for the beginning of
+sequence. The token used is the `cls_token`.
+
+</Tip>
+</parameters>
+<returns>
+
+List of [input IDs](../glossary.html#input-ids) with the appropriate special tokens.
+
+</returns>
+
+<returntype> `List[int]`</returntype>
+<raises>
+- ``ValuError`` -- this value error will be raised on wrong input type.
+</raises>
+<raisederrors>``ValuError``</raisederrors>
+"""
+
+TEST_DOCSTRING_WITH_EXAMPLE = """Constructs a BERTweet tokenizer, using Byte-Pair-Encoding.
+
+This tokenizer inherits from [`~transformers.PreTrainedTokenizer`] which contains most of the main methods.
+Users should refer to this superclass for more information regarding those methods.
+
+<parameters>
+
+- **vocab_file** (`str`) --
+ Path to the vocabulary file.
+- **merges_file** (`str`) --
+ Path to the merges file.
+- **normalization** (`bool`, _optional_, defaults to `False`) --
+ Whether or not to apply a normalization preprocess.
+
+<Tip>
+
+When building a sequence using special tokens, this is not the token that is used for the beginning of
+sequence. The token used is the `cls_token`.
+
+</Tip>
+</parameters>
+<returns>
+
+List of [input IDs](../glossary.html#input-ids) with the appropriate special tokens.
+
+<exampletitle>Example:</exampletitle>
+<example>
+```python
+import transformers
+```
+<example>
+
+</returns>
+
+<returntype> `List[int]`</returntype>"""
+
+
+TEST_DOCSTRING_WITH_PARAM_GROUPS = """
+Builds something very cool!
+
+<parameters>
+
+- **param_a** (`str`) --
+ First default parameter
+- **param_b** (`int`) --
+ Second default parameter
+
+> New group with cool parameters!
+
+- **cool_param_a** (`str`) --
+ First cool parameter
+- **cool_param_b** (`int`) --
+ Second cool parameter
+
+</parameters>
+"""
+
+
+class AutodocTester(unittest.TestCase):
+ test_source_link = (
+ f"https://github.com/huggingface/transformers/blob/main/src/transformers/utils/generic.py#L{TEST_LINE_NUMBER}"
+ )
+ test_source_link_init = f"https://github.com/huggingface/transformers/blob/main/src/transformers/pipelines/__init__.py#L{TEST_LINE_NUMBER2}"
+ test_source_link_timm = (
+ f"https://github.com/rwightman/pytorch-image-models/blob/main/timm/models/factory.py#L{TEST_LINE_NUMBER_TIMM}"
+ )
+
+ def test_find_object_in_package(self):
+ self.assertEqual(find_object_in_package("BertModel", transformers), BertModel)
+ self.assertEqual(find_object_in_package("transformers.BertModel", transformers), BertModel)
+ self.assertEqual(find_object_in_package("models.bert.BertModel", transformers), BertModel)
+ self.assertEqual(find_object_in_package("transformers.models.bert.BertModel", transformers), BertModel)
+ self.assertEqual(find_object_in_package("models.bert.modeling_bert.BertModel", transformers), BertModel)
+ self.assertEqual(
+ find_object_in_package("transformers.models.bert.modeling_bert.BertModel", transformers), BertModel
+ )
+
+ # Works on methods too
+ self.assertEqual(find_object_in_package("BertModel.forward", transformers), BertModel.forward)
+
+ # Test with an object not in the module
+ self.assertIsNone(find_object_in_package("Dataset", transformers))
+
+ def test_remove_example_tags(self):
+ text = "<example>aaa</example>bbb\n<exampletitle>ccc</exampletitle>\n\n<example>ddd</example>"
+ self.assertEqual(remove_example_tags(text), "aaabbb\nccc\n\nddd")
+
+ def test_get_shortest_path(self):
+ self.assertEqual(get_shortest_path(BertModel, transformers), "transformers.BertModel")
+ self.assertEqual(get_shortest_path(BertModel.forward, transformers), "transformers.BertModel.forward")
+ self.assertEqual(get_shortest_path(PushToHubMixin, transformers), "transformers.utils.PushToHubMixin")
+
+ def test_get_type_name(self):
+ self.assertEqual(get_type_name(str), "str")
+ self.assertEqual(get_type_name(BertModel), "BertModel")
+ # Objects from typing which are the most annoying
+ self.assertEqual(get_type_name(List[str]), "typing.List[str]")
+ self.assertEqual(get_type_name(Optional[str]), "typing.Optional[str]")
+ self.assertEqual(get_type_name(Union[bool, int]), "typing.Union[bool, int]")
+ self.assertEqual(get_type_name(List[Optional[str]]), "typing.List[typing.Optional[str]]")
+
+ def test_format_signature(self):
+ self.assertEqual(
+ format_signature(BertModel),
+ [{"name": "config", "val": ""}, {"name": "add_pooling_layer", "val": " = True"}],
+ )
+
+ def func_with_annot(a: int = 1, b: float = 0):
+ pass
+
+ self.assertEqual(
+ format_signature(func_with_annot), [{"name": "a", "val": ": int = 1"}, {"name": "b", "val": ": float = 0"}]
+ )
+
+ def generic_func(*args, **kwargs):
+ pass
+
+ self.assertEqual(
+ format_signature(generic_func), [{"name": "*args", "val": ""}, {"name": "**kwargs", "val": ""}]
+ )
+
+ def test_get_signature_component(self):
+ name = "class transformers.BertweetTokenizer"
+ anchor = "transformers.BertweetTokenizer"
+ signature = [
+ {"name": "vocab_file", "val": ""},
+ {"name": "normalization", "val": " = False"},
+ {"name": "bos_token", "val": " = '<s>'"},
+ ]
+ object_doc = TEST_DOCSTRING
+ source_link = "test_link"
+ expected_signature_component = '<docstring><name>class transformers.BertweetTokenizer</name><anchor>transformers.BertweetTokenizer</anchor><source>test_link</source><parameters>[{"name": "vocab_file", "val": ""}, {"name": "normalization", "val": " = False"}, {"name": "bos_token", "val": " = \'<s>\'"}]</parameters><paramsdesc>- **vocab_file** (`str`) --\n Path to the vocabulary file.\n- **merges_file** (`str`) --\n Path to the merges file.\n- **normalization** (`bool`, _optional_, defaults to `False`) --\n Whether or not to apply a normalization preprocess.\n\n<Tip>\n\nWhen building a sequence using special tokens, this is not the token that is used for the beginning of\nsequence. The token used is the `cls_token`.\n\n</Tip></paramsdesc><paramgroups>0</paramgroups><rettype>`List[int]`</rettype><retdesc>List of [input IDs](../glossary.html#input-ids) with the appropriate special tokens.</retdesc><raises>- ``ValuError`` -- this value error will be raised on wrong input type.</raises><raisederrors>``ValuError``</raisederrors></docstring>\nConstructs a BERTweet tokenizer, using Byte-Pair-Encoding.\n\nThis tokenizer inherits from [`~transformers.PreTrainedTokenizer`] which contains most of the main methods.\nUsers should refer to this superclass for more information regarding those methods.\n\n\n\n\n\n\n\n\n'
+ self.assertEqual(
+ get_signature_component(name, anchor, signature, object_doc, source_link), expected_signature_component
+ )
+
+ name = "class transformers.BertweetTokenizer"
+ anchor = "transformers.BertweetTokenizer"
+ signature = [
+ {"name": "vocab_file", "val": ""},
+ {"name": "normalization", "val": " = False"},
+ {"name": "bos_token", "val": " = '<s>'"},
+ ]
+ object_doc_without_params_and_return = """Constructs a BERTweet tokenizer, using Byte-Pair-Encoding.
+
+This tokenizer inherits from [`~transformers.PreTrainedTokenizer`] which contains most of the main methods.
+Users should refer to this superclass for more information regarding those methods.
+"""
+ expected_signature_component = '<docstring><name>class transformers.BertweetTokenizer</name><anchor>transformers.BertweetTokenizer</anchor><source>test_link</source><parameters>[{"name": "vocab_file", "val": ""}, {"name": "normalization", "val": " = False"}, {"name": "bos_token", "val": " = \'<s>\'"}]</parameters></docstring>\nConstructs a BERTweet tokenizer, using Byte-Pair-Encoding.\n\nThis tokenizer inherits from [`~transformers.PreTrainedTokenizer`] which contains most of the main methods.\nUsers should refer to this superclass for more information regarding those methods.\n\n'
+ self.assertEqual(
+ get_signature_component(name, anchor, signature, object_doc_without_params_and_return, source_link),
+ expected_signature_component,
+ )
+
+ name = "class transformers.cool_function"
+ anchor = "transformers.cool_function"
+ signature = [
+ {"name": "param_a", "val": ""},
+ {"name": "param_b", "val": ""},
+ {"name": "cool_param_a", "val": ""},
+ {"name": "cool_param_b", "val": ""},
+ ]
+ object_doc = TEST_DOCSTRING_WITH_PARAM_GROUPS
+ source_link = "test_link"
+ expected_signature_component = '<docstring><name>class transformers.cool_function</name><anchor>transformers.cool_function</anchor><source>test_link</source><parameters>[{"name": "param_a", "val": ""}, {"name": "param_b", "val": ""}, {"name": "cool_param_a", "val": ""}, {"name": "cool_param_b", "val": ""}]</parameters><paramsdesc>- **param_a** (`str`) --\n First default parameter\n- **param_b** (`int`) --\n Second default parameter\n\n</paramsdesc><paramsdesc1title>New group with cool parameters!</paramsdesc1title><paramsdesc1>\n\n- **cool_param_a** (`str`) --\n First cool parameter\n- **cool_param_b** (`int`) --\n Second cool parameter</paramsdesc1><paramgroups>1</paramgroups></docstring>\n\nBuilds something very cool!\n\n\n\n'
+ self.assertEqual(
+ get_signature_component(name, anchor, signature, object_doc, source_link), expected_signature_component
+ )
+
+ def test_get_source_link(self):
+ page_info = {"package_name": "transformers"}
+ self.assertEqual(get_source_link(transformers.utils.ModelOutput, page_info), self.test_source_link)
+ self.assertEqual(get_source_link(transformers.pipeline, page_info), self.test_source_link_init)
+
+ def test_get_source_link_different_repo_owner(self):
+ page_info = {"package_name": "timm", "repo_owner": "rwightman", "repo_name": "pytorch-image-models"}
+ self.assertEqual(
+ get_source_link(timm.create_model, page_info, version_tag_suffix=""), self.test_source_link_timm
+ )
+
+ def test_document_object(self):
+ page_info = {"package_name": "transformers"}
+
+ model_output_doc = """
+<docstring><name>class transformers.utils.ModelOutput</name><anchor>transformers.utils.ModelOutput</anchor><source>"""
+ model_output_doc += f"{self.test_source_link}"
+ model_output_doc += """</source><parameters>""</parameters></docstring>
+
+Base class for all model outputs as dataclass. Has a `__getitem__` that allows indexing by integer or slice (like a
+tuple) or strings (like a dictionary) that will ignore the `None` attributes. Otherwise behaves like a regular
+python dictionary.
+
+<Tip warning={true}>
+
+You can't unpack a `ModelOutput` directly. Use the [`~utils.ModelOutput.to_tuple`] method to convert it to a tuple
+before.
+
+</Tip>
+
+
+"""
+ self.assertEqual(document_object("utils.ModelOutput", transformers, page_info)[0], model_output_doc)
+
+ def test_find_document_methods(self):
+ self.assertListEqual(find_documented_methods(BertModel), ["forward"])
+ self.assertListEqual(
+ find_documented_methods(BertTokenizer),
+ [
+ "build_inputs_with_special_tokens",
+ "convert_tokens_to_string",
+ "create_token_type_ids_from_sequences",
+ "get_special_tokens_mask",
+ ],
+ )
+ self.assertListEqual(
+ find_documented_methods(BertTokenizerFast),
+ ["build_inputs_with_special_tokens", "create_token_type_ids_from_sequences"],
+ )
+
+ def test_autodoc_return_anchors(self):
+ _, anchors, _ = autodoc("BertTokenizer", transformers, return_anchors=True)
+ self.assertListEqual(
+ anchors,
+ [
+ "transformers.BertTokenizer",
+ "transformers.BertTokenizer.build_inputs_with_special_tokens",
+ "transformers.BertTokenizer.convert_tokens_to_string",
+ "transformers.BertTokenizer.create_token_type_ids_from_sequences",
+ "transformers.BertTokenizer.get_special_tokens_mask",
+ ],
+ )
+
+ _, anchors, _ = autodoc("BertTokenizer", transformers, methods=["__call__", "all"], return_anchors=True)
+ self.assertListEqual(
+ anchors,
+ [
+ "transformers.BertTokenizer",
+ ("transformers.BertTokenizer.__call__", "transformers.PreTrainedTokenizerBase.__call__"),
+ "transformers.BertTokenizer.build_inputs_with_special_tokens",
+ "transformers.BertTokenizer.convert_tokens_to_string",
+ "transformers.BertTokenizer.create_token_type_ids_from_sequences",
+ "transformers.BertTokenizer.get_special_tokens_mask",
+ ],
+ )
+
+ _, anchors, _ = autodoc("BertTokenizer", transformers, methods=["__call__"], return_anchors=True)
+ self.assertListEqual(
+ anchors,
+ [
+ "transformers.BertTokenizer",
+ ("transformers.BertTokenizer.__call__", "transformers.PreTrainedTokenizerBase.__call__"),
+ ],
+ )
+
+ def test_resolve_links_in_text(self):
+ page_info = {"package_name": "transformers"}
+ small_mapping = {
+ "transformers.BertModel": "model_doc/bert.html",
+ "transformers.BertModel.forward": "model_doc/bert.html",
+ "transformers.BertTokenizer": "bert.html",
+ }
+
+ self.maxDiff = None
+ self.assertEqual(
+ resolve_links_in_text(
+ "Link to [`BertModel`], [`BertModel.forward`] and [`BertTokenizer`] as well as [`SomeClass`].",
+ transformers,
+ small_mapping,
+ page_info,
+ ),
+ (
+ "Link to [BertModel](/docs/transformers/main/en/model_doc/bert.html#transformers.BertModel), "
+ "[BertModel.forward()](/docs/transformers/main/en/model_doc/bert.html#transformers.BertModel.forward) "
+ "and [BertTokenizer](/docs/transformers/main/en/bert.html#transformers.BertTokenizer) as well as `SomeClass`."
+ ),
+ )
+
+ self.assertEqual(
+ resolve_links_in_text(
+ "Link to [`~transformers.BertModel`], [`~transformers.BertModel.forward`].",
+ transformers,
+ small_mapping,
+ page_info,
+ ),
+ (
+ "Link to [BertModel](/docs/transformers/main/en/model_doc/bert.html#transformers.BertModel), "
+ "[forward()](/docs/transformers/main/en/model_doc/bert.html#transformers.BertModel.forward)."
+ ),
+ )
+
+ self.assertEqual(
+ resolve_links_in_text(
+ "Link to [`transformers.BertModel`], [`transformers.BertModel.forward`].",
+ transformers,
+ small_mapping,
+ page_info,
+ ),
+ (
+ "Link to [transformers.BertModel](/docs/transformers/main/en/model_doc/bert.html#transformers.BertModel), "
+ "[transformers.BertModel.forward()](/docs/transformers/main/en/model_doc/bert.html#transformers.BertModel.forward)."
+ ),
+ )
+
+ self.assertEqual(
+ resolve_links_in_text(
+ "Link to [`transformers.BertModel.forward#input_ids`], [`~transformers.BertModel.forward#input_ids`].",
+ transformers,
+ small_mapping,
+ page_info,
+ ),
+ (
+ "Link to [input_ids](/docs/transformers/main/en/model_doc/bert.html#transformers.BertModel.forward.input_ids), [input_ids](/docs/transformers/main/en/model_doc/bert.html#transformers.BertModel.forward.input_ids)."
+ ),
+ )
+
+ self.assertEqual(
+ resolve_links_in_text(
+ "This is a regular [`link`](url)",
+ transformers,
+ small_mapping,
+ page_info,
+ ),
+ "This is a regular [`link`](url)",
+ )
+
+ def test_resolve_links_in_text_custom_version_lang(self):
+ page_info = {"package_name": "transformers", "version": "v4.10.0", "language": "fr"}
+ small_mapping = {
+ "transformers.BertModel": "model_doc/bert.html",
+ "transformers.BertModel.forward": "model_doc/bert.html",
+ "transformers.BertTokenizer": "bert.html",
+ }
+
+ self.maxDiff = None
+ self.assertEqual(
+ resolve_links_in_text(
+ "Link to [`BertModel`], [`BertModel.forward`] and [`BertTokenizer`] as well as [`SomeClass`].",
+ transformers,
+ small_mapping,
+ page_info,
+ ),
+ (
+ "Link to [BertModel](/docs/transformers/v4.10.0/fr/model_doc/bert.html#transformers.BertModel), "
+ "[BertModel.forward()](/docs/transformers/v4.10.0/fr/model_doc/bert.html#transformers.BertModel.forward) "
+ "and [BertTokenizer](/docs/transformers/v4.10.0/fr/bert.html#transformers.BertTokenizer) as well as `SomeClass`."
+ ),
+ )
+
+ self.assertEqual(
+ resolve_links_in_text(
+ "Link to [`~transformers.BertModel`], [`~transformers.BertModel.forward`].",
+ transformers,
+ small_mapping,
+ page_info,
+ ),
+ (
+ "Link to [BertModel](/docs/transformers/v4.10.0/fr/model_doc/bert.html#transformers.BertModel), "
+ "[forward()](/docs/transformers/v4.10.0/fr/model_doc/bert.html#transformers.BertModel.forward)."
+ ),
+ )
+
+ self.assertEqual(
+ resolve_links_in_text(
+ "Link to [`transformers.BertModel`], [`transformers.BertModel.forward`].",
+ transformers,
+ small_mapping,
+ page_info,
+ ),
+ (
+ "Link to [transformers.BertModel](/docs/transformers/v4.10.0/fr/model_doc/bert.html#transformers.BertModel), "
+ "[transformers.BertModel.forward()](/docs/transformers/v4.10.0/fr/model_doc/bert.html#transformers.BertModel.forward)."
+ ),
+ )
+
+ def test_is_dataclass_autodoc(self):
+ # example auto generated doc from dataclass
+ @dataclass(frozen=True)
+ class MyClass:
+ attr1: str = "audio_file_path"
+ attr2: str = "transcription"
+
+ self.assertEqual(MyClass.__doc__, "MyClass(attr1: str = 'audio_file_path', attr2: str = 'transcription')")
+
+ # test data class auto generated doc
+ @dataclass(frozen=True)
+ class AutomaticSpeechRecognition:
+ audio_file_path_column: str = "audio_file_path"
+ transcription_column: str = "transcription"
+
+ self.assertTrue(is_dataclass_autodoc(AutomaticSpeechRecognition))
+
+ # test data class auto non-generated doc
+ @dataclass(frozen=True)
+ class AutomaticSpeechRecognition:
+ """
+ Non auto generated doc
+ """
+
+ audio_file_path_column: str = "audio_file_path"
+ transcription_column: str = "transcription"
+
+ self.assertFalse(is_dataclass_autodoc(AutomaticSpeechRecognition))
+
+ # test class with no signature (because of `dict` inheritance)
+ class AutomaticSpeechRecognition(dict):
+ audio_file_path_column: str = "audio_file_path"
+ transcription_column: str = "transcription"
+
+ self.assertFalse(is_dataclass_autodoc(AutomaticSpeechRecognition))
+
+ def test_resolve_links_in_text_other_docs(self):
+ page_info = {"package_name": "transformers", "version": "main", "language": "en"}
+ self.assertEqual(
+ resolve_links_in_text(
+ "Link to [`~accelerate.Accelerator`], [`~accelerate.Accelerator.prepare`].",
+ transformers,
+ {},
+ page_info,
+ ),
+ (
+ "Link to [Accelerator](https://huggingface.co/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator), "
+ "[prepare](https://huggingface.co/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator.prepare)."
+ ),
+ )
+ self.assertEqual(
+ resolve_links_in_text(
+ "Link to [`datasets.Dataset`].",
+ transformers,
+ {},
+ page_info,
+ ),
+ (
+ "Link to [datasets.Dataset](https://huggingface.co/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset)."
+ ),
+ )
+
+ def test_autodoc_getset_descriptor(self):
+ import tokenizers
+
+ documentation = autodoc("AddedToken.content", tokenizers, return_anchors=False)
+ expected_documentation = """<div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8">
+
+<docstring><name>content</name><anchor>None</anchor><parameters>[]</parameters><isgetsetdescriptor></docstring>
+Get the content of this `AddedToken`
+
+</div>\n"""
+ self.assertEqual(documentation, expected_documentation)
+
+ def test_hashlink_example_codeblock(self):
+ dummy_anchor = "myfunc"
+ # test canonical
+ original_md = """Example:
+```python
+import numpy as np
+```"""
+ expected_conversion = """<ExampleCodeBlock anchor="myfunc.example">
+
+Example:
+```python
+import numpy as np
+```
+
+</ExampleCodeBlock>"""
+ self.assertEqual(hashlink_example_codeblock(original_md, dummy_anchor), expected_conversion)
+
+ # test `Examples` ending in `s`
+ original_md = """Examples:
+```python
+import numpy as np
+```"""
+ expected_conversion = """<ExampleCodeBlock anchor="myfunc.example">
+
+Examples:
+```python
+import numpy as np
+```
+
+</ExampleCodeBlock>"""
+ self.assertEqual(hashlink_example_codeblock(original_md, dummy_anchor), expected_conversion)
+
+ # test part of bigger doc description
+ original_md = """Some description about this function
+Example:
+```python
+import numpy as np
+```"""
+ expected_conversion = """Some description about this function
+<ExampleCodeBlock anchor="myfunc.example">
+
+Example:
+```python
+import numpy as np
+```
+
+</ExampleCodeBlock>"""
+ self.assertEqual(hashlink_example_codeblock(original_md, dummy_anchor), expected_conversion)
+
+ # test complex example introduction
+ original_md = """Here is a classification example:
+```python
+import numpy as np
+```"""
+ expected_conversion = """<ExampleCodeBlock anchor="myfunc.example">
+
+Here is a classification example:
+```python
+import numpy as np
+```
+
+</ExampleCodeBlock>"""
+ self.assertEqual(hashlink_example_codeblock(original_md, dummy_anchor), expected_conversion)
+
+ # test doc description with multiple examples
+ original_md = """Here is a classification example:
+```python
+import numpy as np
+```
+
+Here is a regression example:
+```python
+import scipy as sp
+```"""
+ expected_conversion = """<ExampleCodeBlock anchor="myfunc.example">
+
+Here is a classification example:
+```python
+import numpy as np
+```
+
+</ExampleCodeBlock>
+
+<ExampleCodeBlock anchor="myfunc.example-2">
+
+Here is a regression example:
+```python
+import scipy as sp
+```
+
+</ExampleCodeBlock>"""
+ self.assertEqual(hashlink_example_codeblock(original_md, dummy_anchor), expected_conversion)
+
+ # test example with inline ``` (inline ``` should be escaped)
+ original_md = """The tokenization method is `<tokens> <eos> <language code>` for source language documents, and ```<language code>
+<tokens> <eos>``` for target language documents.
+
+Examples:
+
+```python
+>>> from transformers import MBartTokenizer
+
+>>> tokenizer = MBartTokenizer.from_pretrained("facebook/mbart-large-en-ro", src_lang="en_XX", tgt_lang="ro_RO")
+>>> example_english_phrase = " UN Chief Says There Is No Military Solution in Syria"
+>>> expected_translation_romanian = "Şeful ONU declară că nu există o soluţie militară în Siria"
+>>> inputs = tokenizer(example_english_phrase, return_tensors="pt")
+>>> with tokenizer.as_target_tokenizer():
+... labels = tokenizer(expected_translation_romanian, return_tensors="pt")
+>>> inputs["labels"] = labels["input_ids"]
+```"""
+ expected_conversion = """The tokenization method is `<tokens> <eos> <language code>` for source language documents, and ```<language code>
+<tokens> <eos>``` for target language documents.
+
+<ExampleCodeBlock anchor="myfunc.example">
+
+Examples:
+
+```python
+>>> from transformers import MBartTokenizer
+
+>>> tokenizer = MBartTokenizer.from_pretrained("facebook/mbart-large-en-ro", src_lang="en_XX", tgt_lang="ro_RO")
+>>> example_english_phrase = " UN Chief Says There Is No Military Solution in Syria"
+>>> expected_translation_romanian = "Şeful ONU declară că nu există o soluţie militară în Siria"
+>>> inputs = tokenizer(example_english_phrase, return_tensors="pt")
+>>> with tokenizer.as_target_tokenizer():
+... labels = tokenizer(expected_translation_romanian, return_tensors="pt")
+>>> inputs["labels"] = labels["input_ids"]
+```
+
+</ExampleCodeBlock>"""
+ self.assertEqual(hashlink_example_codeblock(original_md, dummy_anchor), expected_conversion)
+
+ # test indentation (there should be no indendetation)
+ original_md = """Some example with indentation
+ ```
+ some pythong
+ ```
+ """
+ self.assertEqual(hashlink_example_codeblock(original_md, dummy_anchor), original_md)
diff --git a/doc-build/third_party/hf-doc-builder/tests/test_build_doc.py b/doc-build/third_party/hf-doc-builder/tests/test_build_doc.py
new file mode 100644
index 00000000..34667617
--- /dev/null
+++ b/doc-build/third_party/hf-doc-builder/tests/test_build_doc.py
@@ -0,0 +1,90 @@
+# coding=utf-8
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from doc_builder.build_doc import _re_autodoc, _re_list_item, generate_frontmatter_in_text, resolve_open_in_colab
+
+
+class BuildDocTester(unittest.TestCase):
+ def test_re_autodoc(self):
+ self.assertEqual(
+ _re_autodoc.search("[[autodoc]] transformers.FlaxBertForQuestionAnswering").groups(),
+ ("transformers.FlaxBertForQuestionAnswering",),
+ )
+
+ def test_re_list_item(self):
+ self.assertEqual(_re_list_item.search(" - forward").groups(), ("forward",))
+
+ def test_resolve_open_in_colab(self):
+ expected = """
+<DocNotebookDropdown
+ classNames="absolute z-10 right-0 top-0"
+ options={[
+ {label: "Mixed", value: "https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/quicktour.ipynb"},
+ {label: "PyTorch", value: "https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/quicktour.ipynb"},
+ {label: "TensorFlow", value: "https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/quicktour.ipynb"},
+ {label: "Mixed", value: "https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/quicktour.ipynb"},
+ {label: "PyTorch", value: "https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/pytorch/quicktour.ipynb"},
+ {label: "TensorFlow", value: "https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/en/tensorflow/quicktour.ipynb"},
+]} />
+"""
+ self.assertEqual(
+ resolve_open_in_colab("\n[[open-in-colab]]\n", {"package_name": "transformers", "page": "quicktour.html"}),
+ expected,
+ )
+
+ def test_generate_frontmatter_in_text(self):
+ # test canonical
+ self.assertEqual(
+ generate_frontmatter_in_text("# Bert\n## BertTokenizer\n### BertTokenizerMethod"),
+ '---\nlocal: bert\nsections:\n- local: berttokenizer\n sections:\n - local: berttokenizermethod\n title: BertTokenizerMethod\n title: BertTokenizer\ntitle: Bert\n---\n<h1 id="bert">Bert</h1>\n<h2 id="berttokenizer">BertTokenizer</h2>\n<h3 id="berttokenizermethod">BertTokenizerMethod</h3>',
+ )
+
+ # test h1 having h3 children (skipping h2 level)
+ self.assertEqual(
+ generate_frontmatter_in_text("# Bert\n### BertTokenizerMethodA\n### BertTokenizerMethodB"),
+ '---\nlocal: bert\nsections:\n- local: berttokenizermethoda\n title: BertTokenizerMethodA\n- local: berttokenizermethodb\n title: BertTokenizerMethodB\ntitle: Bert\n---\n<h1 id="bert">Bert</h1>\n<h3 id="berttokenizermethoda">BertTokenizerMethodA</h3>\n<h3 id="berttokenizermethodb">BertTokenizerMethodB</h3>',
+ )
+
+ # skip python comments in code blocks (because markdown `#` is same as python comment `#`)
+ self.assertEqual(
+ generate_frontmatter_in_text("# Bert\n```\n# python comment\n```\n## BertTokenizer"),
+ '---\nlocal: bert\nsections:\n- local: berttokenizer\n title: BertTokenizer\ntitle: Bert\n---\n<h1 id="bert">Bert</h1>\n```\n# python comment\n```\n<h2 id="berttokenizer">BertTokenizer</h2>',
+ )
+
+ # test header with multiple words
+ self.assertEqual(
+ generate_frontmatter_in_text("# Bert and Bart\n```\n# python comment\n```\n## BertTokenizer"),
+ '---\nlocal: bert-and-bart\nsections:\n- local: berttokenizer\n title: BertTokenizer\ntitle: Bert and Bart\n---\n<h1 id="bert-and-bart">Bert and Bart</h1>\n```\n# python comment\n```\n<h2 id="berttokenizer">BertTokenizer</h2>',
+ )
+
+ # test header with HF emoji
+ self.assertEqual(
+ generate_frontmatter_in_text("# SomeHeader 🤗\n```\n"),
+ '---\nlocal: someheader\ntitle: SomeHeader 🤗\n---\n<h1 id="someheader">SomeHeader 🤗</h1>\n```\n',
+ )
+
+ # test headers with existing ids
+ self.assertEqual(
+ generate_frontmatter_in_text("# Bert[[id1]]\n## BertTokenizer[[id2]]\n### BertTokenizerMethod"),
+ '---\nlocal: id1\nsections:\n- local: id2\n sections:\n - local: berttokenizermethod\n title: BertTokenizerMethod\n title: BertTokenizer\ntitle: Bert\n---\n<h1 id="id1">Bert</h1>\n<h2 id="id2">BertTokenizer</h2>\n<h3 id="berttokenizermethod">BertTokenizerMethod</h3>',
+ )
+
+ # test headers with numbers
+ self.assertEqual(
+ generate_frontmatter_in_text("# Bert 1\n## BertTokenizer 2 3\n### BertTokenizer 4 5 6 Method"),
+ '---\nlocal: bert-1\nsections:\n- local: berttokenizer-2-3\n sections:\n - local: berttokenizer-4-5-6-method\n title: BertTokenizer 4 5 6 Method\n title: BertTokenizer 2 3\ntitle: Bert 1\n---\n<h1 id="bert-1">Bert 1</h1>\n<h2 id="berttokenizer-2-3">BertTokenizer 2 3</h2>\n<h3 id="berttokenizer-4-5-6-method">BertTokenizer 4 5 6 Method</h3>',
+ )
diff --git a/doc-build/third_party/hf-doc-builder/tests/test_convert_doc_file.py b/doc-build/third_party/hf-doc-builder/tests/test_convert_doc_file.py
new file mode 100644
index 00000000..77eb9f1d
--- /dev/null
+++ b/doc-build/third_party/hf-doc-builder/tests/test_convert_doc_file.py
@@ -0,0 +1,24 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from doc_builder.commands.convert_doc_file import shorten_internal_refs
+
+
+class ConvertDocFileTester(unittest.TestCase):
+ def test_shorten_internal_refs(self):
+ self.assertEqual(shorten_internal_refs("Checkout the [`~transformers.Trainer`]."), "Checkout the [`Trainer`].")
+ self.assertEqual(
+ shorten_internal_refs("Look at the [`~transformers.PreTrainedModel.generate`] method."),
+ "Look at the [`~PreTrainedModel.generate`] method.",
+ )
diff --git a/doc-build/third_party/hf-doc-builder/tests/test_convert_md_to_mdx.py b/doc-build/third_party/hf-doc-builder/tests/test_convert_md_to_mdx.py
new file mode 100644
index 00000000..bdb8002b
--- /dev/null
+++ b/doc-build/third_party/hf-doc-builder/tests/test_convert_md_to_mdx.py
@@ -0,0 +1,206 @@
+# coding=utf-8
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import unittest
+from pathlib import Path
+
+from doc_builder.convert_md_to_mdx import (
+ convert_img_links,
+ convert_literalinclude,
+ convert_md_to_mdx,
+ convert_special_chars,
+ process_md,
+)
+
+
+class ConvertMdToMdxTester(unittest.TestCase):
+ def test_convert_md_to_mdx(self):
+ page_info = {"package_name": "transformers", "version": "v4.10.0", "language": "fr"}
+ md_text = "Lorem ipsum dolor sit amet, consectetur adipiscing elit"
+ expected_conversion = """<script lang="ts">
+import {onMount} from "svelte";
+import Tip from "$lib/Tip.svelte";
+import Youtube from "$lib/Youtube.svelte";
+import Docstring from "$lib/Docstring.svelte";
+import CodeBlock from "$lib/CodeBlock.svelte";
+import CodeBlockFw from "$lib/CodeBlockFw.svelte";
+import DocNotebookDropdown from "$lib/DocNotebookDropdown.svelte";
+import CourseFloatingBanner from "$lib/CourseFloatingBanner.svelte";
+import IconCopyLink from "$lib/IconCopyLink.svelte";
+import FrameworkContent from "$lib/FrameworkContent.svelte";
+import Markdown from "$lib/Markdown.svelte";
+import Question from "$lib/Question.svelte";
+import FrameworkSwitchCourse from "$lib/FrameworkSwitchCourse.svelte";
+import InferenceApi from "$lib/InferenceApi.svelte";
+import TokenizersLanguageContent from "$lib/TokenizersLanguageContent.svelte";
+import ExampleCodeBlock from "$lib/ExampleCodeBlock.svelte";
+import Added from "$lib/Added.svelte";
+import Changed from "$lib/Changed.svelte";
+import Deprecated from "$lib/Deprecated.svelte";
+import PipelineIcon from "$lib/PipelineIcon.svelte";
+import PipelineTag from "$lib/PipelineTag.svelte";
+let fw: "pt" | "tf" = "pt";
+onMount(() => {
+ const urlParams = new URLSearchParams(window.location.search);
+ fw = urlParams.get("fw") || "pt";
+});
+</script>
+<svelte:head>
+ <meta name="hf:doc:metadata" content={JSON.stringify(metadata)} >
+</svelte:head>
+Lorem ipsum dolor sit amet, consectetur adipiscing elit"""
+ self.assertEqual(convert_md_to_mdx(md_text, page_info), expected_conversion)
+
+ def test_convert_special_chars(self):
+ self.assertEqual(convert_special_chars("{ lala }"), "{ lala }")
+ self.assertEqual(convert_special_chars("< blo"), "< blo")
+ self.assertEqual(convert_special_chars("<source></source>"), "<source></source>")
+ self.assertEqual(convert_special_chars("<Youtube id='my_vid' />"), "<Youtube id='my_vid' />")
+
+ longer_test = """<script lang="ts">
+import Tip from "$lib/Tip.svelte";
+import Youtube from "$lib/Youtube.svelte";
+import Docstring from "$lib/Docstring.svelte";
+import CodeBlock from "$lib/CodeBlock.svelte";
+export let fw: "pt" | "tf"
+</script>"""
+ self.assertEqual(convert_special_chars(longer_test), longer_test)
+
+ nested_test = """<blockquote>
+ sometext
+ <blockquote>
+ sometext
+ </blockquote>
+</blockquote>"""
+ self.assertEqual(convert_special_chars(nested_test), nested_test)
+
+ html_code = '<a href="Some URl">some_text</a>'
+ self.assertEqual(convert_special_chars(html_code), html_code)
+
+ inner_less = """<blockquote>
+ sometext 4 < 5
+</blockquote>"""
+ self.assertEqual(convert_special_chars(inner_less), inner_less)
+
+ img_code = '<img src="someSrc">'
+ self.assertEqual(convert_special_chars(img_code), img_code)
+
+ video_code = '<video src="someSrc">'
+ self.assertEqual(convert_special_chars(video_code), video_code)
+
+ comment = "<!-- comment -->"
+ self.assertEqual(convert_special_chars(comment), comment)
+
+ def test_convert_img_links(self):
+ page_info = {"package_name": "transformers", "version": "v4.10.0", "language": "fr"}
+
+ img_md = "[img](/imgs/img.gif)"
+ self.assertEqual(convert_img_links(img_md, page_info), "[img](/docs/transformers/v4.10.0/fr/imgs/img.gif)")
+
+ img_html = '<img src="/imgs/img.gif"/>'
+ self.assertEqual(
+ convert_img_links(img_html, page_info), '<img src="/docs/transformers/v4.10.0/fr/imgs/img.gif"/>'
+ )
+
+ def test_process_md(self):
+ page_info = {"package_name": "transformers", "version": "v4.10.0", "language": "fr"}
+
+ text = """[img](/imgs/img.gif)
+{}
+<>"""
+ expected_conversion = """[img](/docs/transformers/v4.10.0/fr/imgs/img.gif)
+{}
+<>"""
+ self.assertEqual(process_md(text, page_info), expected_conversion)
+
+ def test_convert_literalinclude(self):
+ path = Path(__file__).resolve()
+ page_info = {"path": path}
+ # test canonical
+ text = """<literalinclude>
+{"path": "./data/convert_literalinclude_dummy.txt",
+"language": "python",
+"start-after": "START python_import",
+"end-before": "END python_import"}
+</literalinclude>"""
+ # test entire file
+ text = """<literalinclude>
+{"path": "./data/convert_literalinclude_dummy.txt",
+"language": "python"}
+</literalinclude>"""
+ expected_conversion = '''```python
+# START python_import_answer
+import scipy as sp
+# END python_import_answer
+
+# START python_import
+import numpy as np
+import pandas as pd
+# END python_import
+
+# START node_import
+import fs
+# END node_import"""
+```'''
+ self.assertEqual(convert_literalinclude(text, page_info), expected_conversion)
+ # test without language
+ text = """<literalinclude>
+{"path": "./data/convert_literalinclude_dummy.txt",
+"start-after": "START python_import",
+"end-before": "END python_import"}
+</literalinclude>"""
+ expected_conversion = """```
+import numpy as np
+import pandas as pd
+```"""
+ self.assertEqual(convert_literalinclude(text, page_info), expected_conversion)
+ # test with indent
+ text = """Some text
+ <literalinclude>
+{"path": "./data/convert_literalinclude_dummy.txt",
+"start-after": "START python_import",
+"end-before": "END python_import"}
+</literalinclude>"""
+ expected_conversion = """Some text
+ ```
+ import numpy as np
+ import pandas as pd
+ ```"""
+ self.assertEqual(convert_literalinclude(text, page_info), expected_conversion)
+ # test with dedent
+ text = """Some text
+ <literalinclude>
+{"path": "./data/convert_literalinclude_dummy.txt",
+"start-after": "START python_import",
+"end-before": "END python_import",
+"dedent": 7}
+</literalinclude>"""
+ expected_conversion = """Some text
+ ```
+ numpy as np
+ pandas as pd
+ ```"""
+ self.assertEqual(convert_literalinclude(text, page_info), expected_conversion)
+ # test tag rstrip
+ text = """<literalinclude>
+{"path": "./data/convert_literalinclude_dummy.txt",
+"start-after": "START node_import",
+"end-before": "END node_import"}
+</literalinclude>"""
+ expected_conversion = """```
+import fs
+```"""
+ self.assertEqual(convert_literalinclude(text, page_info), expected_conversion)
diff --git a/doc-build/third_party/hf-doc-builder/tests/test_convert_rst_to_mdx.py b/doc-build/third_party/hf-doc-builder/tests/test_convert_rst_to_mdx.py
new file mode 100644
index 00000000..d11dcbe8
--- /dev/null
+++ b/doc-build/third_party/hf-doc-builder/tests/test_convert_rst_to_mdx.py
@@ -0,0 +1,777 @@
+# coding=utf-8
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import unittest
+
+from doc_builder.convert_rst_to_mdx import (
+ _re_anchor_section,
+ _re_args,
+ _re_block,
+ _re_block_info,
+ _re_doc_with_description,
+ _re_double_backquotes,
+ _re_example,
+ _re_func_class,
+ _re_ignore_line_table,
+ _re_ignore_line_table1,
+ _re_links,
+ _re_math,
+ _re_obj,
+ _re_prefix_links,
+ _re_ref_with_description,
+ _re_returns,
+ _re_rst_option,
+ _re_sep_line_table,
+ _re_simple_doc,
+ _re_simple_ref,
+ _re_single_backquotes,
+ apply_min_indent,
+ convert_rst_blocks,
+ convert_rst_formatting,
+ convert_rst_links,
+ convert_special_chars,
+ find_indent,
+ is_empty_line,
+ parse_options,
+ parse_rst_docstring,
+ process_titles,
+ remove_indent,
+ split_arg_line,
+ split_pt_tf_code_blocks,
+ split_raise_line,
+ split_return_line,
+)
+
+
+class ConvertRstToMdxTester(unittest.TestCase):
+ def test_re_obj(self):
+ self.assertEqual(_re_obj.search("There is an :obj:`object`.").groups(), ("object",))
+ self.assertIsNone(_re_obj.search("There is no :obj:`object."))
+
+ def test_re_math(self):
+ self.assertEqual(_re_math.search("There is a :math:`formula`.").groups(), ("formula",))
+ self.assertIsNone(_re_math.search("There is no :math:`formula."))
+
+ def test_re_single_backquotes(self):
+ self.assertEqual(_re_single_backquotes.search("There is some `code`.").groups(), (" ", "code", "."))
+ self.assertIsNone(_re_single_backquotes.search("This is double ``code``."))
+ self.assertEqual(_re_single_backquotes.search("`code` at the beginning.").groups(), ("", "code", " "))
+ self.assertEqual(_re_single_backquotes.search("At the end there is some `code`").groups(), (" ", "code", ""))
+
+ def test_re_double_backquotes(self):
+ self.assertEqual(_re_double_backquotes.search("There is some ``code``.").groups(), (" ", "code", "."))
+ self.assertIsNone(_re_double_backquotes.search("This is single `code`."))
+ self.assertIsNone(_re_double_backquotes.search("This is triple ```code```."))
+ self.assertEqual(_re_double_backquotes.search("``code`` at the beginning.").groups(), ("", "code", " "))
+ self.assertEqual(_re_double_backquotes.search("At the end there is some ``code``").groups(), (" ", "code", ""))
+
+ def test_re_func_class(self):
+ self.assertEqual(_re_func_class.search("There is a :class:`class`.").groups(), ("class",))
+ self.assertEqual(_re_func_class.search("There is a :func:`function`.").groups(), ("function",))
+ self.assertEqual(_re_func_class.search("There is a :meth:`method`.").groups(), ("method",))
+ self.assertIsNone(_re_func_class.search("There is no :obj:`object`."))
+
+ def test_re_links(self):
+ self.assertEqual(_re_links.search("This is a regular `link <url>`_").groups(), ("link", "url"))
+ self.assertEqual(_re_links.search("This is a regular `link <url>`__").groups(), ("link", "url"))
+
+ def test_re_prefix_links(self):
+ self.assertEqual(
+ _re_prefix_links.search("This is a regular :prefix_link:`link <url>`").groups(),
+ ("link", "url"),
+ )
+
+ def test_re_simple_doc(self):
+ self.assertEqual(_re_simple_doc.search("This is a link to an inner page :doc:`page`.").groups(), ("page",))
+
+ def test_re_doc_with_description(self):
+ self.assertEqual(
+ _re_doc_with_description.search("This is a link to an inner page :doc:`page name <page ref>`.").groups(),
+ ("page name", "page ref"),
+ )
+
+ def test_re_simple_ref(self):
+ self.assertEqual(
+ _re_simple_ref.search("This is a link to an inner section :ref:`section`.").groups(), ("section",)
+ )
+
+ def test_re_ref_with_description(self):
+ self.assertEqual(
+ _re_ref_with_description.search(
+ "This is a link to an inner section :ref:`section name <section ref>`."
+ ).groups(),
+ ("section name", "section ref"),
+ )
+
+ def test_re_example(self):
+ self.assertEqual(_re_example.search(" Example::").groups(), ("Example",))
+ self.assertEqual(_re_example.search(" Generation example::").groups(), ("Generation example",))
+ self.assertIsNone(_re_example.search(" Return:"))
+
+ def test_re_block(self):
+ self.assertEqual(_re_block.search(" .. note::").groups(), ("note",))
+ self.assertEqual(_re_block.search(" .. code-example::").groups(), ("code-example",))
+
+ def test_re_block_info(self):
+ self.assertEqual(_re_block_info.search(" .. note:: python").groups(), ("python",))
+ self.assertEqual(_re_block_info.search(" .. code-example:: python").groups(), ("python",))
+
+ def test_re_rst_option(self):
+ self.assertEqual(_re_rst_option.search(" :size: 123").groups(), ("size", " 123"))
+ self.assertEqual(_re_rst_option.search(" :members: func1, func2,").groups(), ("members", " func1, func2,"))
+ self.assertEqual(
+ _re_rst_option.search(
+ " :special-members: __call__, batch_decode, decode, encode, push_to_hub"
+ ).groups(),
+ ("special-members", " __call__, batch_decode, decode, encode, push_to_hub"),
+ )
+
+ def test_re_args(self):
+ self.assertIsNotNone(_re_args.search(" Args:"))
+ self.assertIsNotNone(_re_args.search(" Arg:"))
+ self.assertIsNone(_re_args.search(" Arg: lala"))
+
+ self.assertIsNotNone(_re_args.search(" Arguments:"))
+ self.assertIsNotNone(_re_args.search(" Argument:"))
+ self.assertIsNone(_re_args.search(" Argument: lala"))
+
+ self.assertIsNotNone(_re_args.search(" Params:"))
+ self.assertIsNotNone(_re_args.search(" Param:"))
+ self.assertIsNone(_re_args.search(" Param: lala"))
+
+ self.assertIsNotNone(_re_args.search(" Parameters:"))
+ self.assertIsNotNone(_re_args.search(" Parameter:"))
+ self.assertIsNone(_re_args.search(" Parameter: lala"))
+
+ self.assertIsNotNone(_re_args.search(" Attributes:"))
+ self.assertIsNotNone(_re_args.search(" Attribute:"))
+ self.assertIsNone(_re_args.search(" Attribute: lala"))
+
+ def test_re_returns(self):
+ self.assertIsNotNone(_re_returns.search(" Returns:"))
+ self.assertIsNotNone(_re_returns.search(" Return:"))
+ self.assertIsNone(_re_returns.search(" Return: lala"))
+
+ def test_re_ignore_line_table(self):
+ self.assertIsNotNone(_re_ignore_line_table.search("+--- + --- +"))
+
+ def test_re_ignore_line_table1(self):
+ self.assertIsNotNone(_re_ignore_line_table1.search("| +--- + --- +"))
+
+ def test_re_sep_line_table(self):
+ self.assertIsNotNone(_re_sep_line_table.search("+=== +===+ === +"))
+
+ def test_re_anchor_table(self):
+ self.assertEqual(_re_anchor_section.search(".. _reference:").groups(), ("reference",))
+
+ def test_convert_rst_formatting(self):
+ test_text_1 = """
+This text comports a bit of everything: `italics`, *italics*, some ``code``. There is some already converted **bold** and
+some references to :obj:`objects`, :class:`~transformers.classes`, :meth:`methods` and :func:`funcs`. Also, we can find
+a :math:`formula`."""
+ expected_converted_1 = """
+This text comports a bit of everything: *italics*, *italics*, some `code`. There is some already converted **bold** and
+some references to `objects`, [`~transformers.classes`], [`methods`] and [`funcs`]. Also, we can find
+a \\\\(formula\\\\)."""
+ self.assertEqual(convert_rst_formatting(test_text_1), expected_converted_1)
+
+ test_text_2 = """
+ This contains some ``code on
+ two lines``.
+"""
+ expected_converted_2 = "\n This contains some `code on two lines`.\n"
+ self.assertEqual(convert_rst_formatting(test_text_2), expected_converted_2)
+
+ test_text_3 = """
+This contains some ``code on
+two lines.`` with more ``on the second line``.
+"""
+ expected_converted_3 = "\nThis contains some `code on two lines.` with more `on the second line`.\n"
+ self.assertEqual(convert_rst_formatting(test_text_3), expected_converted_3)
+
+ test_text_4 = """
+This contains some ``code on
+two lines.`` with more ``on the
+third line``.
+"""
+ expected_converted_4 = "\nThis contains some `code on two lines.` with more `on the third line`.\n"
+ self.assertEqual(convert_rst_formatting(test_text_4), expected_converted_4)
+
+ def test_convert_rst_links(self):
+ page_info = {"package_name": "transformers"}
+ self.assertEqual(
+ convert_rst_links("This is a regular `link <url>`_", page_info), "This is a regular [link](url)"
+ )
+ self.assertEqual(
+ convert_rst_links("This is a regular `link <url>`__", page_info), "This is a regular [link](url)"
+ )
+
+ self.assertEqual(
+ convert_rst_links("This is a prefixed :prefix_link:`link <url>`", page_info),
+ "This is a prefixed [link](https://github.com/huggingface/transformers/tree/main/url)",
+ )
+ self.assertEqual(
+ convert_rst_links("This is a prefixed :prefix_link:`link <url>`", page_info),
+ "This is a prefixed [link](https://github.com/huggingface/transformers/tree/main/url)",
+ )
+
+ self.assertEqual(
+ convert_rst_links("This is a link to an inner page :doc:`page`.", page_info),
+ "This is a link to an inner page [page](/docs/transformers/main/en/page).",
+ )
+ self.assertEqual(
+ convert_rst_links("This is a link to an inner page :doc:`page name <page ref>`.", page_info),
+ "This is a link to an inner page [page name](/docs/transformers/main/en/page ref).",
+ )
+
+ self.assertEqual(
+ convert_rst_links("This is a link to an inner section :ref:`section`.", page_info),
+ "This is a link to an inner section [section](#section).",
+ )
+ self.assertEqual(
+ convert_rst_links("This is a link to an inner section :ref:`section name <section ref>`.", page_info),
+ "This is a link to an inner section [section name](#section ref).",
+ )
+
+ page_info["page"] = "model_doc/bert.html"
+ self.assertEqual(
+ convert_rst_links("This is a link to an inner section :ref:`section`.", page_info),
+ "This is a link to an inner section [section](/docs/transformers/main/en/model_doc/bert#section).",
+ )
+ self.assertEqual(
+ convert_rst_links("This is a link to an inner section :ref:`section name <section ref>`.", page_info),
+ "This is a link to an inner section [section name](/docs/transformers/main/en/model_doc/bert#section ref).",
+ )
+
+ def test_convert_rst_links_with_version_and_lang(self):
+ page_info = {"package_name": "transformers", "version": "v4.10.0", "language": "fr"}
+
+ self.assertEqual(
+ convert_rst_links("This is a link to an inner page :doc:`page`.", page_info),
+ "This is a link to an inner page [page](/docs/transformers/v4.10.0/fr/page).",
+ )
+ self.assertEqual(
+ convert_rst_links("This is a link to an inner page :doc:`page name <page ref>`.", page_info),
+ "This is a link to an inner page [page name](/docs/transformers/v4.10.0/fr/page ref).",
+ )
+
+ self.assertEqual(
+ convert_rst_links("This is a link to an inner section :ref:`section`.", page_info),
+ "This is a link to an inner section [section](#section).",
+ )
+ self.assertEqual(
+ convert_rst_links("This is a link to an inner section :ref:`section name <section ref>`.", page_info),
+ "This is a link to an inner section [section name](#section ref).",
+ )
+
+ page_info["page"] = "model_doc/bert.html"
+ self.assertEqual(
+ convert_rst_links("This is a link to an inner section :ref:`section`.", page_info),
+ "This is a link to an inner section [section](/docs/transformers/v4.10.0/fr/model_doc/bert#section).",
+ )
+ self.assertEqual(
+ convert_rst_links("This is a link to an inner section :ref:`section name <section ref>`.", page_info),
+ "This is a link to an inner section [section name](/docs/transformers/v4.10.0/fr/model_doc/bert#section ref).",
+ )
+ self.assertEqual(
+ convert_rst_links("`What are input IDs? <../glossary.html#input-ids>`__", page_info),
+ "[What are input IDs?](../glossary#input-ids)",
+ )
+
+ def test_is_empty_line(self):
+ self.assertTrue(is_empty_line(""))
+ self.assertTrue(is_empty_line(" "))
+ self.assertFalse(is_empty_line("a"))
+ self.assertFalse(is_empty_line(" a"))
+
+ def test_find_indent(self):
+ self.assertEqual(find_indent(""), 0)
+ self.assertEqual(find_indent("a"), 0)
+ self.assertEqual(find_indent(" "), 3)
+ self.assertEqual(find_indent(" a"), 3)
+
+ def test_convert_special_chars(self):
+ self.assertEqual(convert_special_chars("{ lala }"), "{ lala }")
+ self.assertEqual(convert_special_chars("< blo"), "< blo")
+ self.assertEqual(convert_special_chars("<source></source>"), "<source></source>")
+ self.assertEqual(convert_special_chars("<Youtube id='my_vid' />"), "<Youtube id='my_vid' />")
+
+ longer_test = """<script lang="ts">
+import Tip from "$lib/Tip.svelte";
+import Youtube from "$lib/Youtube.svelte";
+import Docstring from "$lib/Docstring.svelte";
+import CodeBlock from "$lib/CodeBlock.svelte";
+export let fw: "pt" | "tf"
+</script>"""
+ self.assertEqual(convert_special_chars(longer_test), longer_test)
+
+ nested_test = """<blockquote>
+ sometext
+ <blockquote>
+ sometext
+ </blockquote>
+</blockquote>"""
+ self.assertEqual(convert_special_chars(nested_test), nested_test)
+
+ html_code = '<a href="Some URl">some_text</a>'
+ self.assertEqual(convert_special_chars(html_code), html_code)
+
+ inner_less = """<blockquote>
+ sometext 4 < 5
+</blockquote>"""
+ self.assertEqual(convert_special_chars(inner_less), inner_less)
+
+ img_code = '<img src="someSrc">'
+ self.assertEqual(convert_special_chars(img_code), img_code)
+
+ def test_parse_options(self):
+ self.assertEqual(
+ parse_options(" :size: 123\n :members: func1, func2,\n func3"),
+ {"size": "123", "members": "func1, func2, func3"},
+ )
+ self.assertEqual(parse_options(" :size: 123\n :members:"), {"size": "123", "members": ""})
+
+ def test_convert_rst_blocks(self):
+ page_info = {"package_name": "transformers", "version": "v4.10.0", "language": "fr"}
+
+ original_rst = """
+text
+text
+
+..
+
+ comment
+
+.. code-block:: python
+
+ print(1 + 1)
+
+Example::
+ example code
+
+.. note::
+ This is a note.
+
+.. autoclass:: transformers.AdamW
+ :members:
+
+.. autoclass:: transformers.PreTrainedTokenizer
+ :special-members: __call__, batch_decode, decode, encode, push_to_hub
+ :members:
+
+.. autoclass:: transformers.BertTokenizer
+ :members: build_inputs_with_special_tokens, get_special_tokens_mask,
+ create_token_type_ids_from_sequences, save_vocabulary
+
+.. autofunction:: transformers.create_optimizer
+
+.. image:: /imgs/warmup_cosine_schedule.png
+ :target: /imgs/warmup_cosine_schedule.png
+ :alt: Alternative text
+
+.. math::
+
+ formula
+"""
+
+ expected_conversion = """
+text
+text
+
+<!--comment
+-->
+
+```python
+print(1 + 1)
+```
+
+<exampletitle>Example:</exampletitle>
+
+<example>```python
+example code
+```
+</example>
+<Tip>
+
+This is a note.
+
+</Tip>
+
+[[autodoc]] transformers.AdamW
+ - all
+
+[[autodoc]] transformers.PreTrainedTokenizer
+ - __call__
+ - batch_decode
+ - decode
+ - encode
+ - push_to_hub
+ - all
+
+[[autodoc]] transformers.BertTokenizer
+ - build_inputs_with_special_tokens
+ - get_special_tokens_mask
+ - create_token_type_ids_from_sequences
+ - save_vocabulary
+
+[[autodoc]] transformers.create_optimizer
+
+<img alt="Alternative text" src="/docs/transformers/v4.10.0/fr/imgs/warmup_cosine_schedule.png"/>
+
+$$formula$$
+"""
+
+ self.assertEqual(convert_rst_blocks(original_rst, page_info), expected_conversion)
+
+ rst_with_indent = """
+ This is inside a docstring, so we have some indent.
+
+ .. note::
+ There is a note in the middle.
+
+ We should keep the indent for the note.
+"""
+ expected_conversion = """
+ This is inside a docstring, so we have some indent.
+
+ <Tip>
+
+ There is a note in the middle.
+
+ </Tip>
+
+ We should keep the indent for the note.
+"""
+ self.assertEqual(convert_rst_blocks(rst_with_indent, page_info), expected_conversion)
+
+ def test_split_return_line(self):
+ self.assertEqual(
+ split_return_line("A :obj:`str` or a :obj:`bool`: the result"),
+ ("A :obj:`str` or a :obj:`bool`", " the result"),
+ )
+ self.assertEqual(
+ split_return_line("A :obj:`str` or a :obj:`bool` the result"),
+ (None, "A :obj:`str` or a :obj:`bool` the result"),
+ )
+ self.assertEqual(split_return_line("A :obj:`str` or a :obj:`bool`:"), ("A :obj:`str` or a :obj:`bool`", ""))
+ self.assertEqual(
+ split_return_line(" :obj:`str` or :obj:`bool`: some result:"),
+ (" :obj:`str` or :obj:`bool`", " some result:"),
+ )
+ self.assertEqual(split_return_line(":class:`IterableDataset`"), (":class:`IterableDataset`", ""))
+ self.assertEqual(split_return_line("`int`"), ("`int`", ""))
+
+ def test_split_raise_line(self):
+ self.assertEqual(split_raise_line("SomeError some error"), ("SomeError", "some error"))
+ self.assertEqual(split_raise_line("SomeError: some error"), ("SomeError", "some error"))
+ self.assertEqual(
+ split_raise_line("[SomeError](https:://someurl): some error"),
+ ("[SomeError](https:://someurl)", "some error"),
+ )
+ self.assertEqual(
+ split_raise_line(
+ "[`HTTPError`](https://2.python-requests.org/en/master/api/#requests.HTTPError) if credentials are invalid"
+ ),
+ (
+ "[`HTTPError`](https://2.python-requests.org/en/master/api/#requests.HTTPError)",
+ "if credentials are invalid",
+ ),
+ )
+
+ def test_split_arg_line(self):
+ self.assertEqual(split_arg_line(" x (:obj:`int`): an int"), (" x (:obj:`int`)", " an int"))
+ self.assertEqual(split_arg_line(" x (:obj:`int`)"), (" x (:obj:`int`)", ""))
+
+ def test_parse_rst_docsting(self):
+ # test canonical
+ rst_docstring = """
+docstring
+
+Args:
+ a (:obj:`str` or :obj:`bool`): some parameter
+ b (:obj:`str` or :obj:`bool`):
+ Another parameter with the description below
+
+Raises:
+ `pa.ArrowInvalidError`: if the arrow data casting fails
+ TypeError: if the target type is not supported according, e.g.
+ - point1
+ - point2
+ [`HTTPError`](https://2.python-requests.org/en/master/api/#requests.HTTPError) if credentials are invalid
+ [`HTTPError`](https://2.python-requests.org/en/master/api/#requests.HTTPError) if connection got lost
+
+Returns:
+ :obj:`str` or :obj:`bool`: some result
+
+Example::
+
+ print(a + b)
+
+End of the arg section.
+"""
+ expected_conversion = """
+docstring
+
+<parameters>
+
+- **a** (:obj:`str` or :obj:`bool`) -- some parameter
+- **b** (:obj:`str` or :obj:`bool`) --
+ Another parameter with the description below
+
+</parameters>
+
+<raises>
+
+- ``pa.ArrowInvalidError`` -- if the arrow data casting fails
+- ``TypeError`` -- if the target type is not supported according, e.g.
+ - point1
+ - point2
+- [`HTTPError`](https://2.python-requests.org/en/master/api/#requests.HTTPError) -- if credentials are invalid
+- [`HTTPError`](https://2.python-requests.org/en/master/api/#requests.HTTPError) -- if connection got lost
+
+</raises>
+
+<raisederrors>``pa.ArrowInvalidError`` or ``TypeError`` or ``HTTPError``</raisederrors>
+
+<returns>
+
+ some result
+
+</returns>
+
+<returntype> :obj:`str` or :obj:`bool`</returntype>
+
+Example::
+
+ print(a + b)
+
+End of the arg section.
+"""
+ self.assertEqual(parse_rst_docstring(rst_docstring), expected_conversion)
+
+ # test yields
+ rst_docstring = """
+docstring
+
+Args:
+ a (:obj:`str` or :obj:`bool`): some parameter
+ b (:obj:`str` or :obj:`bool`):
+ Another parameter with the description below
+
+Yields:
+ :obj:`str` or :obj:`bool`: some result
+"""
+ expected_conversion = """
+docstring
+
+<parameters>
+
+- **a** (:obj:`str` or :obj:`bool`) -- some parameter
+- **b** (:obj:`str` or :obj:`bool`) --
+ Another parameter with the description below
+
+</parameters>
+
+<yields>
+
+ some result
+
+</yields>
+
+<yieldtype> :obj:`str` or :obj:`bool`</yieldtype>
+"""
+
+ self.assertEqual(parse_rst_docstring(rst_docstring), expected_conversion)
+
+ # test multiple parameter blocks
+ rst_docstring = """Args:
+ a (:obj:`str` or :obj:`bool`): some parameter
+ b (:obj:`str` or :obj:`bool`):
+ Another parameter with the description below
+
+Parameters:
+ a (:obj:`str` or :obj:`bool`): some parameter
+ b (:obj:`str` or :obj:`bool`):
+ Another parameter with the description below
+"""
+ expected_conversion = """
+
+
+
+<parameters>- **a** (:obj:`str` or :obj:`bool`) -- some parameter
+- **b** (:obj:`str` or :obj:`bool`) --
+ Another parameter with the description below
+- **a** (:obj:`str` or :obj:`bool`) -- some parameter
+- **b** (:obj:`str` or :obj:`bool`) --
+ Another parameter with the description below</parameters>
+"""
+ self.assertEqual(parse_rst_docstring(rst_docstring), expected_conversion)
+
+ def test_remove_indent(self):
+ example1 = """
+ Lala
+ Loulou
+
+ - This is a list.
+ This item is long.
+ - This is the second item.
+ - This list is nested
+ - With two items.
+ Now we are at the nested level
+
+ - We return to the previous level.
+
+ Now we are out of the list.
+"""
+ expected1 = """
+Lala
+Loulou
+
+- This is a list.
+ This item is long.
+- This is the second item.
+ - This list is nested
+ - With two items.
+ Now we are at the nested level
+
+- We return to the previous level.
+
+Now we are out of the list.
+"""
+
+ example1b = """
+ Lala
+ Loulou
+
+ - This is a list.
+ This item is long.
+ - This is the second item.
+ - This list is nested
+ - With two items.
+ Now we are at the nested level
+
+ - We return to the previous level.
+
+ Now we are out of the list.
+"""
+ expected1b = """
+Lala
+Loulou
+
+- This is a list.
+ This item is long.
+- This is the second item.
+ - This list is nested
+ - With two items.
+ Now we are at the nested level
+
+- We return to the previous level.
+
+Now we are out of the list.
+"""
+
+ example2 = """
+[[autodoc]] transformers.BertModel
+ - forward
+
+ [[autodoc]] transformers.function
+
+[[autodoc]] transformers.BertTokenizer
+ - __call__
+ - all
+"""
+ expected2 = """
+[[autodoc]] transformers.BertModel
+ - forward
+
+[[autodoc]] transformers.function
+
+[[autodoc]] transformers.BertTokenizer
+ - __call__
+ - all
+"""
+
+ example3 = """
+ Lala
+
+ ```python
+ def function(x):
+ return x
+ ```
+
+ Loulou
+"""
+ expected3 = """
+Lala
+
+```python
+def function(x):
+ return x
+```
+
+Loulou
+"""
+ self.assertEqual(remove_indent(example1), expected1)
+ self.assertEqual(remove_indent(example1b), expected1b)
+ self.assertEqual(remove_indent(example2), expected2)
+ self.assertEqual(remove_indent(example3), expected3)
+
+ def test_process_titles(self):
+ self.assertListEqual(
+ process_titles(["Title 1", "=======", "text", "", "section", "--------"]),
+ ["# Title 1", "text", "", "## section"],
+ )
+
+ def test_split_pt_tf_code_blocks(self):
+ content = """
+bla
+
+```py
+common_code
+## PYTORCH CODE
+pytorch_code
+## TENSORFLOW CODE
+tf_code
+```
+
+bli
+"""
+
+ expected = """
+bla
+
+<frameworkcontent>
+<pt>
+```py
+common_code
+pytorch_code
+```
+</pt>
+<tf>
+```py
+common_code
+tf_code
+```
+</tf>
+</frameworkcontent>
+
+bli
+"""
+
+ self.assertEqual(split_pt_tf_code_blocks(content), expected)
+
+ def test_apply_min_indent(self):
+ self.assertEqual(apply_min_indent("aaa\n bb\n\n ccc\ndd", 4), " aaa\n bb\n\n ccc\n dd")
diff --git a/doc-build/third_party/hf-doc-builder/tests/test_convert_to_notebook.py b/doc-build/third_party/hf-doc-builder/tests/test_convert_to_notebook.py
new file mode 100644
index 00000000..13e1e0ab
--- /dev/null
+++ b/doc-build/third_party/hf-doc-builder/tests/test_convert_to_notebook.py
@@ -0,0 +1,176 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from doc_builder.convert_to_notebook import (
+ _re_copyright,
+ _re_header,
+ _re_math_delimiter,
+ _re_python_code,
+ _re_youtube,
+ expand_links,
+ parse_input_output,
+ split_frameworks,
+)
+
+
+class ConvertToNotebookTester(unittest.TestCase):
+ def test_re_math_delimiter(self):
+ self.assertEqual(_re_math_delimiter.search("\\\\(lala\\\\)").groups()[0], "lala")
+ self.assertListEqual(_re_math_delimiter.findall("\\\\(lala\\\\)xx\\\\(loulou\\\\)"), ["lala", "loulou"])
+
+ def test_re_copyright(self):
+ self.assertIsNotNone(
+ _re_copyright.search("<!--Copyright 2021 Hugging Face\n more more more\n--> rest of text")
+ )
+
+ def test_re_youtube(self):
+ self.assertEqual(_re_youtube.search('<Youtube id="tiZFewofSLM"/>').groups()[0], "tiZFewofSLM")
+
+ def test_re_header(self):
+ self.assertIsNotNone(_re_header.search("# Title"))
+ self.assertIsNotNone(_re_header.search("### Subesection"))
+ self.assertIsNone(_re_header.search("Title"))
+
+ def test_re_python(self):
+ self.assertIsNotNone(_re_python_code.search("```py"))
+ self.assertIsNotNone(_re_python_code.search("```python"))
+ self.assertIsNone(_re_python_code.search("```bash"))
+ self.assertIsNone(_re_python_code.search("```"))
+
+ def test_parse_inputs_output(self):
+ expected = "from transformers import pipeline\n\nclassifier = pipeline('sentiment-analysis')"
+ doctest_lines_no_output = [
+ ">>> from transformers import pipeline",
+ "",
+ ">>> classifier = pipeline('sentiment-analysis')",
+ ]
+ doctest_lines_with_output = [
+ ">>> from transformers import pipeline",
+ "",
+ ">>> classifier = pipeline('sentiment-analysis')",
+ "output",
+ ]
+ regular_lines = ["from transformers import pipeline", "", "classifier = pipeline('sentiment-analysis')"]
+
+ self.assertListEqual(parse_input_output(regular_lines), [(expected, None)])
+ self.assertListEqual(parse_input_output(doctest_lines_no_output), [(expected, None)])
+ self.assertListEqual(parse_input_output(doctest_lines_with_output), [(expected, "output")])
+
+ def test_parse_inputs_output_multiple_outputs(self):
+ expected_1 = "from transformers import pipeline"
+ expected_2 = "classifier = pipeline('sentiment-analysis')"
+
+ doctest_lines_with_output = [
+ ">>> from transformers import pipeline",
+ "output 1",
+ ">>> classifier = pipeline('sentiment-analysis')",
+ "output 2",
+ ]
+
+ self.assertListEqual(
+ parse_input_output(doctest_lines_with_output), [(expected_1, "output 1"), (expected_2, "output 2")]
+ )
+
+ doctest_lines_with_one_output = [
+ ">>> from transformers import pipeline",
+ "output 1",
+ ">>> classifier = pipeline('sentiment-analysis')",
+ ]
+
+ self.assertListEqual(
+ parse_input_output(doctest_lines_with_one_output), [(expected_1, "output 1"), (expected_2, None)]
+ )
+
+ def test_split_framewors(self):
+ test_content = """
+Intro
+```py
+common_code_sample
+```
+Content
+<frameworkcontent>
+<pt>
+```py
+pt_sample
+```
+</pt>
+<tf>
+```py
+tf_sample
+```
+</tf>
+</frameworkcontent>
+End
+"""
+ mixed_content = """
+Intro
+```py
+common_code_sample
+```
+Content
+```py
+pt_sample
+```
+```py
+tf_sample
+```
+End
+"""
+ pt_content = """
+Intro
+```py
+common_code_sample
+```
+Content
+```py
+pt_sample
+```
+End
+"""
+ tf_content = """
+Intro
+```py
+common_code_sample
+```
+Content
+```py
+tf_sample
+```
+End
+"""
+ for expected, obtained in zip([mixed_content, pt_content, tf_content], split_frameworks(test_content)):
+ self.assertEqual(expected, obtained)
+
+ def test_expand_links(self):
+ page_info = {"package_name": "transformers", "page": "quicktour.html"}
+ self.assertEqual(
+ expand_links("Checkout the [task summary](task-summary)", page_info),
+ "Checkout the [task summary](https://huggingface.co/docs/transformers/main/en/task-summary)",
+ )
+ self.assertEqual(
+ expand_links("Checkout the [`Trainer`](/docs/transformers/main/en/trainer#Trainer)", page_info),
+ "Checkout the [`Trainer`](https://huggingface.co/docs/transformers/main/en/trainer#Trainer)",
+ )
+
+ page_info = {"package_name": "datasets", "page": "quicktour.html", "version": "stable", "language": "fr"}
+ self.assertEqual(
+ expand_links("Checkout the [task summary](task-summary)", page_info),
+ "Checkout the [task summary](https://huggingface.co/docs/datasets/stable/fr/task-summary)",
+ )
+
+ page_info = {"package_name": "transformers", "page": "data/quicktour.html"}
+ self.assertEqual(
+ expand_links("Checkout the [task summary](task-summary)", page_info),
+ "Checkout the [task summary](https://huggingface.co/docs/transformers/main/en/data/task-summary)",
+ )
diff --git a/doc-build/third_party/hf-doc-builder/tests/test_style_doc.py b/doc-build/third_party/hf-doc-builder/tests/test_style_doc.py
new file mode 100644
index 00000000..465a2a48
--- /dev/null
+++ b/doc-build/third_party/hf-doc-builder/tests/test_style_doc.py
@@ -0,0 +1,179 @@
+# coding=utf-8
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import unittest
+
+from doc_builder.style_doc import (
+ _re_code,
+ _re_docstyle_ignore,
+ _re_list,
+ _re_tip,
+ format_code_example,
+ format_text,
+ parse_code_example,
+ style_docstring,
+)
+
+
+class BuildDocTester(unittest.TestCase):
+ def test_re_code(self):
+ self.assertEqual(_re_code.search("```").groups(), ("", ""))
+ self.assertEqual(_re_code.search("```python").groups(), ("", "python"))
+ self.assertEqual(_re_code.search(" ```").groups(), (" ", ""))
+ self.assertEqual(_re_code.search(" ```python").groups(), (" ", "python"))
+
+ def test_re_docstyle_ignore(self):
+ self.assertIsNotNone(_re_docstyle_ignore.search("# docstyle-ignore"))
+ self.assertIsNotNone(_re_docstyle_ignore.search(" # docstyle-ignore"))
+
+ def test_re_list(self):
+ self.assertEqual(_re_list.search(" - normal list").groups(), (" - ",))
+ self.assertEqual(_re_list.search("* star list").groups(), ("* ",))
+ self.assertEqual(_re_list.search(" 1. digit list").groups(), (" 1. ",))
+
+ def test_re_tip(self):
+ self.assertIsNotNone(_re_tip.search("<Tip>"))
+ self.assertIsNotNone(_re_tip.search(" <Tip>"))
+
+ self.assertIsNotNone(_re_tip.search("</Tip>"))
+ self.assertIsNotNone(_re_tip.search(" </Tip>"))
+
+ self.assertIsNotNone(_re_tip.search("<Tip warning={true}>"))
+ self.assertIsNotNone(_re_tip.search(" <Tip warning={true}>"))
+
+ def test_parse_code_example(self):
+ # One code sample, no output
+ self.assertEqual(parse_code_example(["code line 1", "code line 2"]), (["code line 1\ncode line 2"], []))
+ self.assertEqual(
+ parse_code_example([">>> code line 1", ">>> code line 2"]), (["code line 1\ncode line 2"], [])
+ )
+ self.assertEqual(
+ parse_code_example([">>> code line 1", "... code line 2"]), (["code line 1\ncode line 2"], [])
+ )
+
+ # One code sample, one output
+ self.assertEqual(
+ parse_code_example([">>> code line 1", ">>> code line 2", "output"]),
+ (["code line 1\ncode line 2"], ["output"]),
+ )
+ self.assertEqual(
+ parse_code_example([">>> code line 1", "... code line 2", "output"]),
+ (["code line 1\ncode line 2"], ["output"]),
+ )
+
+ # Two code samples, one output
+ self.assertEqual(
+ parse_code_example([">>> code sample 1", "output 1", ">>> code sample 2"]),
+ (["code sample 1", "code sample 2"], ["output 1"]),
+ )
+ self.assertEqual(
+ parse_code_example([">>> code sample 1", "... sample 1 other line", "output 1", ">>> code sample 2"]),
+ (["code sample 1\nsample 1 other line", "code sample 2"], ["output 1"]),
+ )
+
+ # Two code samples, two outputs
+ self.assertEqual(
+ parse_code_example([">>> code sample 1", "output 1", ">>> code sample 2", "output 2"]),
+ (["code sample 1", "code sample 2"], ["output 1", "output 2"]),
+ )
+ self.assertEqual(
+ parse_code_example(
+ [">>> code sample 1", "... indented code", "output 1", ">>> code sample 2", "output 2"]
+ ),
+ (["code sample 1\n indented code", "code sample 2"], ["output 1", "output 2"]),
+ )
+
+ def test_format_code_example(self):
+ no_output_code = "from transformers import AutoModel\nmodel = AutoModel('bert-base-cased')"
+ expected_result = 'from transformers import AutoModel\n\nmodel = AutoModel("bert-base-cased")'
+ self.assertEqual(format_code_example(no_output_code, max_len=119), (expected_result, ""))
+
+ no_output_code = ">>> from transformers import AutoModel\n>>> model = AutoModel('bert-base-cased')"
+ expected_result = '>>> from transformers import AutoModel\n\n>>> model = AutoModel("bert-base-cased")'
+ self.assertEqual(format_code_example(no_output_code, max_len=119), (expected_result, ""))
+
+ no_output_code_with_indent = (
+ " >>> from transformers import AutoModel\n >>> model = AutoModel('bert-base-cased')"
+ )
+ expected_result = ' >>> from transformers import AutoModel\n\n >>> model = AutoModel("bert-base-cased")'
+ self.assertEqual(format_code_example(no_output_code_with_indent, max_len=119), (expected_result, ""))
+
+ no_output_code_with_error = "from transformers import AutoModel\nmodel = AutoModel('bert-base-cased'"
+ result, error = format_code_example(no_output_code_with_error, max_len=119)
+ self.assertEqual(result, no_output_code_with_error)
+ self.assertIn("Error message:\nCannot parse", error)
+
+ code_with_output = ">>> from transformers import AutoModel\n>>> model = AutoModel('bert-base-cased')\noutput"
+ expected_result = '>>> from transformers import AutoModel\n\n>>> model = AutoModel("bert-base-cased")\noutput'
+ self.assertEqual(format_code_example(code_with_output, max_len=119), (expected_result, ""))
+
+ def test_format_text(self):
+ text = "This is an example text that will \nbe used in\n these examples. "
+ clean_text = re.sub("\s+", " ", text).strip()
+ for max_len in [20, 30, 50, 100]:
+ formatted_text = format_text(text, max_len=max_len)
+ # Nothing was lost
+ self.assertEqual(clean_text, formatted_text.replace("\n", " "))
+ for line in formatted_text.split("\n"):
+ self.assertTrue(len(line) <= max_len)
+
+ formatted_text = format_text(text, max_len=max_len, min_indent=4)
+ # Nothing was lost
+ clean_formatted_text = "\n".join([l[4:] for l in formatted_text.split("\n")])
+ self.assertEqual(clean_text, clean_formatted_text.replace("\n", " "))
+ for line in formatted_text.split("\n"):
+ self.assertTrue(line.startswith(" "))
+ self.assertTrue(len(line) <= max_len)
+
+ formatted_text = format_text(text, max_len=max_len, prefix="- ", min_indent=4)
+ # Nothing was lost
+ clean_formatted_text = "\n".join([l[4:] for l in formatted_text.split("\n")])
+ self.assertEqual(clean_text, clean_formatted_text.replace("\n", " "))
+ for i, line in enumerate(formatted_text.split("\n")):
+ self.assertTrue(line.startswith(" " if i > 0 else " - "))
+ self.assertTrue(len(line) <= max_len)
+
+ def test_format_docstring_empty_line(self):
+ test_docstring = """Function description
+
+Params:
+
+ x (`int`): This is x.
+ y (`float`): this is y.
+"""
+ expected_result = """Function description
+
+Params:
+ x (`int`): This is x.
+ y (`float`): this is y.
+"""
+
+ self.assertEqual(style_docstring(test_docstring, 119)[0], expected_result)
+
+ def test_format_docstring_handle_params_without_empty_line_after_description(self):
+ test_docstring = """Function description
+Params:
+ x (`int`): This is x.
+ y (`float`): this is y.
+"""
+ expected_result = """Function description
+
+Params:
+ x (`int`): This is x.
+ y (`float`): this is y.
+"""
+
+ self.assertEqual(style_docstring(test_docstring, 119)[0], expected_result)
diff --git a/doc-build/third_party/hf-doc-builder/tests/test_utils.py b/doc-build/third_party/hf-doc-builder/tests/test_utils.py
new file mode 100644
index 00000000..93ff28d2
--- /dev/null
+++ b/doc-build/third_party/hf-doc-builder/tests/test_utils.py
@@ -0,0 +1,68 @@
+# coding=utf-8
+# Copyright 2021 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import tempfile
+import unittest
+from pathlib import Path
+
+import yaml
+from doc_builder.utils import update_versions_file
+
+
+class UtilsTester(unittest.TestCase):
+ def test_update_versions_file(self):
+ repo_folder = Path(__file__).parent.parent
+ # test canonical
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ with open(f"{tmp_dir}/_versions.yml", "w") as tmp_yml:
+ versions = [{"version": "main"}, {"version": "v4.2.3"}, {"version": "v4.2.1"}]
+ yaml.dump(versions, tmp_yml)
+ update_versions_file(tmp_dir, "v4.2.2", repo_folder)
+ with open(f"{tmp_dir}/_versions.yml", "r") as tmp_yml:
+ yml_str = tmp_yml.read()
+ expected_yml = "- version: main\n- version: v4.2.3\n- version: v4.2.2\n- version: v4.2.1\n"
+ self.assertEqual(yml_str, expected_yml)
+
+ # test yml with main version only
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ with open(f"{tmp_dir}/_versions.yml", "w") as tmp_yml:
+ versions = [{"version": "main"}]
+ yaml.dump(versions, tmp_yml)
+ update_versions_file(tmp_dir, "v4.2.2", repo_folder)
+ with open(f"{tmp_dir}/_versions.yml", "r") as tmp_yml:
+ yml_str = tmp_yml.read()
+ expected_yml = "- version: main\n- version: v4.2.2\n"
+ self.assertEqual(yml_str, expected_yml)
+
+ # test yml without main version
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ with open(f"{tmp_dir}/_versions.yml", "w") as tmp_yml:
+ versions = [{"version": "v4.2.2"}]
+ yaml.dump(versions, tmp_yml)
+
+ self.assertRaises(ValueError, update_versions_file, tmp_dir, "v4.2.2", repo_folder)
+
+ # test inserting duplicate version into yml
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ with open(f"{tmp_dir}/_versions.yml", "w") as tmp_yml:
+ versions = [{"version": "main"}]
+ yaml.dump(versions, tmp_yml)
+ update_versions_file(tmp_dir, "v4.2.2", repo_folder)
+ update_versions_file(tmp_dir, "v4.2.2", repo_folder) # inserting duplicate version
+ with open(f"{tmp_dir}/_versions.yml", "r") as tmp_yml:
+ yml_str = tmp_yml.read()
+ expected_yml = "- version: main\n- version: v4.2.2\n"
+ self.assertEqual(yml_str, expected_yml)
diff --git a/docusaurus/docusaurus.config.js b/docusaurus/docusaurus.config.js
index 0e9a82a6..3ab088b0 100644
--- a/docusaurus/docusaurus.config.js
+++ b/docusaurus/docusaurus.config.js
@@ -179,8 +179,7 @@ const config = {
},
{
label: 'Examples',
- to:
- 'https://github.com/hpcaitech/ColossalAI/tree/main/examples',
+ to: 'https://github.com/hpcaitech/ColossalAI/tree/main/examples',
},
{
label: 'Forum',
diff --git a/docusaurus/src/components/Docstring/index.js b/docusaurus/src/components/Docstring/index.js
new file mode 100644
index 00000000..4e6cd8b1
--- /dev/null
+++ b/docusaurus/src/components/Docstring/index.js
@@ -0,0 +1,101 @@
+import React from 'react';
+import ReactMarkdown from 'react-markdown';
+import './styles.css';
+import CodeBlock from '@theme/CodeBlock';
+
+export function DocStringContainer(props) {
+ return <div className="docstring-container">{props.children}</div>;
+}
+
+export function Signature(props) {
+ return (
+ <div className="signature">
+ {'('}
+ {props.children}
+ {')'}
+ </div>
+ );
+}
+
+export function Divider(props) {
+ return <h3 className="divider">{props.name}</h3>;
+}
+
+export function Parameters(props) {
+ return (
+ <div>
+ <Divider name="Parameters" />
+ <ReactMarkdown>{props.children}</ReactMarkdown>
+ </div>
+ );
+}
+
+export function Returns(props) {
+ return (
+ <div>
+ <Divider name="Returns" />
+ <ReactMarkdown>{`${props.name}: ${props.desc}`}</ReactMarkdown>
+ </div>
+ );
+}
+
+export function Yields(props) {
+ return (
+ <div>
+ <Divider name="Yields" />
+ <ReactMarkdown>{`${props.name}: ${props.desc}`}</ReactMarkdown>
+ </div>
+ );
+}
+
+export function Raises(props) {
+ return (
+ <div>
+ <Divider name="Raises" />
+ <ReactMarkdown>{`${props.name}: ${props.desc}`}</ReactMarkdown>
+ </div>
+ );
+}
+
+export function Title(props) {
+ return (
+ <div className="title-container">
+ <div className="title-module">
+ <h3>{props.type}</h3> <h2>{props.name}</h2>
+ </div>
+ <div className="title-source">
+ {'<'}
+ <a href={props.source} className="title-source">
+ source
+ </a>
+ {'>'}
+ </div>
+ </div>
+ );
+}
+
+export function ObjectDoc(props) {
+ return (
+ <div>
+ <Divider name="Doc" />
+ <ReactMarkdown>{props.doc}</ReactMarkdown>
+ </div>
+ );
+}
+
+export function ExampleCode(props) {
+ return (
+ <div>
+ <Divider name="Example" />
+ <ReactMarkdown>{props.code}</ReactMarkdown>
+ </div>
+ );
+}
+
+export default function TestComponent() {
+ return (
+ <DocStringContainer>
+ <div className="test">hey</div>
+ </DocStringContainer>
+ );
+}
diff --git a/docusaurus/src/components/Docstring/styles.css b/docusaurus/src/components/Docstring/styles.css
new file mode 100644
index 00000000..52acd671
--- /dev/null
+++ b/docusaurus/src/components/Docstring/styles.css
@@ -0,0 +1,59 @@
+.docstring-container {
+ margin-top: 2rem;
+ margin-bottom: 2rem;
+ padding: 1rem;
+ border-color: #818cf8;
+ border-style: solid;
+ border-width: 2px;
+ border-radius: 0.5rem;
+}
+
+.signature {
+ color: #6b7280;
+ font-weight: bold;
+ margin-bottom: 0.5rem;
+}
+
+.title-container {
+ display: flex;
+ flex-direction: row;
+ justify-content: space-between;
+}
+
+.title-source {
+ color: #6b7280;
+}
+
+.title-module {
+ display: flex;
+ flex-direction: row;
+ align-items: center;
+ padding: 5px;
+ color: #4f46e5;
+}
+
+/* divider */
+.divider {
+ display: flex;
+ align-items: center;
+ text-align: center;
+ font-size: large;
+ color: #4f46e5;
+ margin-bottom: 1rem;
+ margin-top: 1rem;
+}
+
+.divider::before,
+.divider::after {
+ content: '';
+ flex: 1;
+ border-bottom: 1px solid #4f46e5;
+}
+
+.divider:not(:empty)::before {
+ margin-right: 0.25em;
+}
+
+.divider:not(:empty)::after {
+ margin-left: 0.25em;
+}
diff --git a/scripts/build.sh b/scripts/build.sh
index d019ada8..7b334cc6 100644
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -8,8 +8,9 @@ DOCUSAURUS_DIR="${SCRIPT_DIR}/../docusaurus"
# build docs
cd "${DOC_BUILD_DIR}"
-doc-build extract -o hpcaitech -p ColossalAI
-doc-build docusaurus -d ../docusaurus
+docer extract -o hpcaitech -p ColossalAI
+docer docusaurus -d ../docusaurus
+docer autodoc -o hpcaitech -p ColossalAI
# build html
cd "${DOCUSAURUS_DIR}"