From 1470d213eead45b44507341a15635e80f1a39fba Mon Sep 17 00:00:00 2001 From: Rafael Vasquez Date: Thu, 17 Mar 2022 20:23:17 -0400 Subject: [PATCH 1/5] Add script to update doc table Signed-off-by: Rafael Vasquez --- Makefile | 5 ++ docs/README.md | 13 +++++ docs/mlx-read-only-deployment.md | 1 - tools/python/update_doc_table.py | 88 ++++++++++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 docs/README.md create mode 100755 tools/python/update_doc_table.py diff --git a/Makefile b/Makefile index 51ab5087..804bdded 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,11 @@ check_doc_links: ## Check markdown files for invalid links @python3 tools/python/verify_doc_links.py @echo "$@: OK" +.PHONY: update_doc_table +update_doc_table: ## Check markdown files for invalid links + @python3 tools/python/update_doc_table.py + @echo "$@: OK" + .PHONY: check_license check_license: ## Make sure source files have license header @git grep -L "SPDX-License-Identifier: Apache-2.0" -- *.py *.yml *.yaml *.sh *.html *.js *.css *.ts *.tsx ':!*.bundle.js' | \ diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..6da60d94 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,13 @@ +!-- Do not edit. This file was generated by ./tools/python/update_doc_table.py (make update_doc_table) --> +| Document | Description | +| --- | --- | +| [import-assets.md](https://github.com/machine-learning-exchange/mlx/blob/master/docs/import-assets.md) | Import Data and AI Assets in MLX Catalog | +| [install-mlx-on-kind.md](https://github.com/machine-learning-exchange/mlx/blob/master/docs/install-mlx-on-kind.md) | Deploy MLX on KIND | +| [install-mlx-on-kubeflow.md](https://github.com/machine-learning-exchange/mlx/blob/master/docs/install-mlx-on-kubeflow.md) | Deploy MLX on an Existing Kubeflow Cluster | +| [mlx-install-operator.md](https://github.com/machine-learning-exchange/mlx/blob/master/docs/mlx-install-operator.md) | Install Operator Manually | +| [mlx-install-with-kubeflow.md](https://github.com/machine-learning-exchange/mlx/blob/master/docs/mlx-install-with-kubeflow.md) | MLX Deployment on Kubernetes or OpenShift (Asset Catalog and Execution Engine) | +| [mlx-read-only-deployment.md](https://github.com/machine-learning-exchange/mlx/blob/master/docs/mlx-read-only-deployment.md) | Deploy the MLX Read-Only mode on an Existing Kubernetes Cluster | +| [mlx-setup.md](https://github.com/machine-learning-exchange/mlx/blob/master/docs/mlx-setup.md) | Deploy MLX on an existing Kubernetes cluster | +| [mlx-workshop.md](https://github.com/machine-learning-exchange/mlx/blob/master/docs/mlx-workshop.md) | MLX Workshop | +| [troubleshooting.md](https://github.com/machine-learning-exchange/mlx/blob/master/docs/troubleshooting.md) | Troubleshooting | +| [usage-steps.md](https://github.com/machine-learning-exchange/mlx/blob/master/docs/usage-steps.md) | Usage Steps | \ No newline at end of file diff --git a/docs/mlx-read-only-deployment.md b/docs/mlx-read-only-deployment.md index 55692aa5..38780ed3 100644 --- a/docs/mlx-read-only-deployment.md +++ b/docs/mlx-read-only-deployment.md @@ -1,4 +1,3 @@ - ## Deploy the MLX Read-Only mode on an Existing Kubernetes Cluster To deploy MLX in read-only mode on an existing Kubernetes cluster without Kubeflow, clone the MLX repo and apply the manifest using Kustomize: diff --git a/tools/python/update_doc_table.py b/tools/python/update_doc_table.py new file mode 100755 index 00000000..ff4b17f0 --- /dev/null +++ b/tools/python/update_doc_table.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 + +# Copyright 2021 The MLX Contributors +# +# SPDX-License-Identifier: Apache-2.0 + +import itertools +import re + +from glob import glob +from os import environ as env +from os.path import abspath, dirname, split + +GITHUB_REPO = env.get( + "GITHUB_REPO", "https://github.com/machine-learning-exchange/mlx/") + +md_file_path_expressions = [ + "/docs/*.md" +] + +script_folder = abspath(dirname(__file__)) +project_root_dir = abspath(dirname(dirname(script_folder))) +github_repo_master_path = "{}/blob/master/docs/".format( + GITHUB_REPO.rstrip("/")) + + +def find_md_files() -> [str]: + + print("Checking for Markdown files here:\n") + for path_expr in md_file_path_expressions: + print(" " + path_expr.lstrip("/")) + print("") + + md_files_list_of_lists = [glob(project_root_dir + path_expr, recursive=True) + for path_expr in md_file_path_expressions] + + return sorted(list(itertools.chain(*md_files_list_of_lists))) + + +def get_header_from_md_file(md_file_path: str) -> [str]: + + with open(md_file_path, "r") as f: + md_file_content = f.readline() + + # remove "#" and newline character + header = re.sub(r"#+ *|\n", "", md_file_content) + + # return description header + return header + + +def update_doc_table() -> [str]: + + # 1. find all relevant Markdown files + md_file_paths = find_md_files() + + # 2. extract all descriptions using headers (first line) from files + descriptions = [ + get_header_from_md_file(file) + for file in md_file_paths + ] + + # 3. format filenames as Markdown hyperlinks: [name](url) + md_filenames = ["[" + split(file)[1] + "](" + github_repo_master_path + split(file)[1] + ")" + for file in md_file_paths] + + table = [] + table.append( + "\n" + "\n" + "| Document | Description |\n" + "| --- | --- |" + ) + + for i in range(len(md_filenames)): + if("README.md" in md_filenames[i]): + continue + table.append( + f"| {md_filenames[i]} | {descriptions[i]} |" + ) + + f = open("./docs/README.md", "w") + f.write("\n".join(table)) + f.close() + + +if __name__ == '__main__': + update_doc_table() From 19d9c1f850e0872234607e51f4b2dc9a1d8e5d0b Mon Sep 17 00:00:00 2001 From: Christian Kadner Date: Wed, 30 Mar 2022 01:09:13 -0700 Subject: [PATCH 2/5] Correct HTML comment Signed-off-by: Rafael Vasquez --- docs/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/README.md b/docs/README.md index 6da60d94..26bdcbe9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,4 +1,5 @@ -!-- Do not edit. This file was generated by ./tools/python/update_doc_table.py (make update_doc_table) --> + + | Document | Description | | --- | --- | | [import-assets.md](https://github.com/machine-learning-exchange/mlx/blob/master/docs/import-assets.md) | Import Data and AI Assets in MLX Catalog | @@ -10,4 +11,4 @@ | [mlx-setup.md](https://github.com/machine-learning-exchange/mlx/blob/master/docs/mlx-setup.md) | Deploy MLX on an existing Kubernetes cluster | | [mlx-workshop.md](https://github.com/machine-learning-exchange/mlx/blob/master/docs/mlx-workshop.md) | MLX Workshop | | [troubleshooting.md](https://github.com/machine-learning-exchange/mlx/blob/master/docs/troubleshooting.md) | Troubleshooting | -| [usage-steps.md](https://github.com/machine-learning-exchange/mlx/blob/master/docs/usage-steps.md) | Usage Steps | \ No newline at end of file +| [usage-steps.md](https://github.com/machine-learning-exchange/mlx/blob/master/docs/usage-steps.md) | Usage Steps | From 523e0871e580eaf621d906e933de8b069ca7310d Mon Sep 17 00:00:00 2001 From: Rafael Vasquez Date: Wed, 30 Mar 2022 11:54:34 -0400 Subject: [PATCH 3/5] Use relative links Signed-off-by: Rafael Vasquez --- docs/README.md | 20 ++++++++++---------- tools/python/update_doc_table.py | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/README.md b/docs/README.md index 26bdcbe9..d8b49d0c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,13 +2,13 @@ | Document | Description | | --- | --- | -| [import-assets.md](https://github.com/machine-learning-exchange/mlx/blob/master/docs/import-assets.md) | Import Data and AI Assets in MLX Catalog | -| [install-mlx-on-kind.md](https://github.com/machine-learning-exchange/mlx/blob/master/docs/install-mlx-on-kind.md) | Deploy MLX on KIND | -| [install-mlx-on-kubeflow.md](https://github.com/machine-learning-exchange/mlx/blob/master/docs/install-mlx-on-kubeflow.md) | Deploy MLX on an Existing Kubeflow Cluster | -| [mlx-install-operator.md](https://github.com/machine-learning-exchange/mlx/blob/master/docs/mlx-install-operator.md) | Install Operator Manually | -| [mlx-install-with-kubeflow.md](https://github.com/machine-learning-exchange/mlx/blob/master/docs/mlx-install-with-kubeflow.md) | MLX Deployment on Kubernetes or OpenShift (Asset Catalog and Execution Engine) | -| [mlx-read-only-deployment.md](https://github.com/machine-learning-exchange/mlx/blob/master/docs/mlx-read-only-deployment.md) | Deploy the MLX Read-Only mode on an Existing Kubernetes Cluster | -| [mlx-setup.md](https://github.com/machine-learning-exchange/mlx/blob/master/docs/mlx-setup.md) | Deploy MLX on an existing Kubernetes cluster | -| [mlx-workshop.md](https://github.com/machine-learning-exchange/mlx/blob/master/docs/mlx-workshop.md) | MLX Workshop | -| [troubleshooting.md](https://github.com/machine-learning-exchange/mlx/blob/master/docs/troubleshooting.md) | Troubleshooting | -| [usage-steps.md](https://github.com/machine-learning-exchange/mlx/blob/master/docs/usage-steps.md) | Usage Steps | +| [import-assets.md](./import-assets.md) | Import Data and AI Assets in MLX Catalog | +| [install-mlx-on-kind.md](./install-mlx-on-kind.md) | Deploy MLX on KIND | +| [install-mlx-on-kubeflow.md](./install-mlx-on-kubeflow.md) | Deploy MLX on an Existing Kubeflow Cluster | +| [mlx-install-operator.md](./mlx-install-operator.md) | Install Operator Manually | +| [mlx-install-with-kubeflow.md](./mlx-install-with-kubeflow.md) | MLX Deployment on Kubernetes or OpenShift (Asset Catalog and Execution Engine) | +| [mlx-read-only-deployment.md](./mlx-read-only-deployment.md) | Deploy the MLX Read-Only mode on an Existing Kubernetes Cluster | +| [mlx-setup.md](./mlx-setup.md) | Deploy MLX on an existing Kubernetes cluster | +| [mlx-workshop.md](./mlx-workshop.md) | MLX Workshop | +| [troubleshooting.md](./troubleshooting.md) | Troubleshooting | +| [usage-steps.md](./usage-steps.md) | Usage Steps | \ No newline at end of file diff --git a/tools/python/update_doc_table.py b/tools/python/update_doc_table.py index ff4b17f0..6cc4c049 100755 --- a/tools/python/update_doc_table.py +++ b/tools/python/update_doc_table.py @@ -61,7 +61,7 @@ def update_doc_table() -> [str]: ] # 3. format filenames as Markdown hyperlinks: [name](url) - md_filenames = ["[" + split(file)[1] + "](" + github_repo_master_path + split(file)[1] + ")" + md_filenames = ["[" + split(file)[1] + "](./" + split(file)[1] + ")" for file in md_file_paths] table = [] From ac41d06c3962cb838122f3dd4cae03572e860e47 Mon Sep 17 00:00:00 2001 From: Rafael Vasquez Date: Wed, 30 Mar 2022 15:36:41 -0400 Subject: [PATCH 4/5] Lint and add description Signed-off-by: Rafael Vasquez --- tools/python/update_doc_table.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tools/python/update_doc_table.py b/tools/python/update_doc_table.py index 6cc4c049..d1f54da2 100755 --- a/tools/python/update_doc_table.py +++ b/tools/python/update_doc_table.py @@ -4,25 +4,23 @@ # # SPDX-License-Identifier: Apache-2.0 +""" + This script is used to (re)generate the /docs/README.md file with a table that describes the + purpose of each document. Run it from the project root folder via `make update_doc_table`. +""" + import itertools import re from glob import glob -from os import environ as env from os.path import abspath, dirname, split -GITHUB_REPO = env.get( - "GITHUB_REPO", "https://github.com/machine-learning-exchange/mlx/") - md_file_path_expressions = [ "/docs/*.md" ] script_folder = abspath(dirname(__file__)) project_root_dir = abspath(dirname(dirname(script_folder))) -github_repo_master_path = "{}/blob/master/docs/".format( - GITHUB_REPO.rstrip("/")) - def find_md_files() -> [str]: From baeb0cda418cc971ab28164b3c2d55fa241e1ec3 Mon Sep 17 00:00:00 2001 From: Rafael Vasquez Date: Wed, 30 Mar 2022 15:44:57 -0400 Subject: [PATCH 5/5] Format function Signed-off-by: Rafael Vasquez --- tools/python/update_doc_table.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/python/update_doc_table.py b/tools/python/update_doc_table.py index d1f54da2..710cb2f6 100755 --- a/tools/python/update_doc_table.py +++ b/tools/python/update_doc_table.py @@ -22,6 +22,7 @@ script_folder = abspath(dirname(__file__)) project_root_dir = abspath(dirname(dirname(script_folder))) + def find_md_files() -> [str]: print("Checking for Markdown files here:\n")