From b0049bddedfae5a42ab133743d87f2825c2a1fac Mon Sep 17 00:00:00 2001 From: merveenoyan Date: Mon, 18 Jul 2022 18:37:44 +0200 Subject: [PATCH 01/62] refactored card class --- skops/card/_model_card.py | 107 ++++++++++++++++++++------------------ 1 file changed, 55 insertions(+), 52 deletions(-) diff --git a/skops/card/_model_card.py b/skops/card/_model_card.py index 49b7f1ac..30f5f676 100644 --- a/skops/card/_model_card.py +++ b/skops/card/_model_card.py @@ -7,55 +7,58 @@ import skops -def _extract_estimator_config(model): - """Extracts estimator configuration and renders them into a vertical table. - - Parameters - ---------- - model (estimator): scikit-learn pipeline or model. - - Returns - ------- - str: - Markdown table of hyperparameters. - """ - hyperparameter_dict = model.get_params(deep=True) - table = "| Hyperparameters | Value |\n| :-- | :-- |\n" - for hyperparameter, value in hyperparameter_dict.items(): - table += f"| {hyperparameter} | {value} |\n" - return table - - -def create_model_card( - model, - card_data, - **card_kwargs, -): - """Creates a model card for the model and saves it to the target directory. - - Parameters: - ---------- - model: estimator - scikit-learn compatible estimator. - card_data: CardData - CardData object. - card_kwargs: - Card kwargs are information you can pass to fill in the sections of the - card template, e.g. model_description, citation_bibtex, get_started_code. - """ - ROOT = skops.__path__ - model_plot = re.sub(r"\n\s+", "", str(estimator_html_repr(model))) - hyperparameter_table = _extract_estimator_config(model) - card_data.library_name = "sklearn" - template_path = card_kwargs.get("template_path") - if template_path is None: - template_path = os.path.join(f"{ROOT[0]}", "card", "default_template.md") - card_kwargs["template_path"] = template_path - card = ModelCard.from_template( - card_data=card_data, - hyperparameter_table=hyperparameter_table, - model_plot=model_plot, - **card_kwargs, - ) - - return card +class Card: + def __init__(self, model): + self.model = model + self.hyperparameters_table = self.extract_estimator_config(self.model) + self.model_plot = re.sub(r"\n\s+", "", str(estimator_html_repr(model))) + + def add(self): + pass + + def add_inspection(self): + pass + + def save(self, path, card_data, **card_kwargs): + """Fills model card template, renders and saves it as markdown file to the target directory. + + Parameters: + ---------- + card_data: CardData + CardData object. + card_kwargs: + Card kwargs are information you can pass to fill in the sections of the + card template, e.g. model_description, citation_bibtex, get_started_code. + """ + ROOT = skops.__path__ + card_data.library_name = "sklearn" + template_path = card_kwargs.get("template_path") + if template_path is None: + template_path = os.path.join(f"{ROOT[0]}", "card", "default_template.md") + card_kwargs["template_path"] = template_path + card = ModelCard.from_template( + card_data=card_data, + hyperparameter_table=self.hyperparameter_table, + model_plot=self.model_plot, + **card_kwargs, + ) + card.save(path) + + def extract_estimator_config(model): + """Extracts estimator configuration and renders them into a vertical table. + + Parameters + ---------- + model (estimator): scikit-learn pipeline or model. + + Returns + ------- + str: + Markdown table of hyperparameters. + """ + + hyperparameter_dict = model.get_params(deep=True) + table = "| Hyperparameters | Value |\n| :-- | :-- |\n" + for hyperparameter, value in hyperparameter_dict.items(): + table += f"| {hyperparameter} | {value} |\n" + return table From 1d34c6fd8cf4f13d55878fddc9d2046eff7a75e7 Mon Sep 17 00:00:00 2001 From: merveenoyan Date: Mon, 18 Jul 2022 19:02:13 +0200 Subject: [PATCH 02/62] updated example --- examples/plot_model_card.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/examples/plot_model_card.py b/examples/plot_model_card.py index 5531d80b..268f0472 100644 --- a/examples/plot_model_card.py +++ b/examples/plot_model_card.py @@ -85,15 +85,9 @@ ) citation = "bibtex\n@inproceedings{...,year={2020}}" -model_card = card.create_model_card( - model, - card_data=card_data, - limitations=limitations, - model_description=model_description, - citation_bibtex=citation, - model_card_authors=model_card_authors, - get_started_code=get_started_code, -) + +model_card = card.Card(model) + _, pkl_name = mkstemp(prefix="skops-", suffix=".pkl") @@ -105,4 +99,12 @@ model=pkl_name, requirements=[f"scikit-learn={sklearn.__version__}"], dst=local_repo ) -model_card.save(os.path.join(f"{local_repo}", "README.md")) +model_card.save( + os.path.join(f"{local_repo}", "README.md"), + card_data=card_data, + limitations=limitations, + model_description=model_description, + citation_bibtex=citation, + model_card_authors=model_card_authors, + get_started_code=get_started_code, +) From e8d27e45cf8378e83ccf9b5648330e8d178ca593 Mon Sep 17 00:00:00 2001 From: merveenoyan Date: Tue, 19 Jul 2022 14:05:20 +0200 Subject: [PATCH 03/62] add function and removed adding sections in save --- examples/plot_model_card.py | 36 ++++++++++++++++-------------------- skops/card/_model_card.py | 25 ++++++++++++++++--------- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/examples/plot_model_card.py b/examples/plot_model_card.py index 268f0472..54dbebff 100644 --- a/examples/plot_model_card.py +++ b/examples/plot_model_card.py @@ -64,12 +64,6 @@ # Then, we pass information other than metadata in kwargs. # We'll initialize a local repository and save the card with the model in it. -limitations = "This model is not ready to be used in production." -model_description = ( - "This is a HistGradientBoostingClassifier model trained on breast cancer dataset." - " It's trained with Halving Grid Search Cross Validation, with parameter grids on" - " max_leaf_nodes and max_depth." -) license = "mit" card_data = CardData( @@ -79,12 +73,6 @@ metrics=["acc"], ) -model_card_authors = "skops_user" -get_started_code = ( - "import pickle\nwith open(dtc_pkl_filename, 'rb') as file:\nclf = pickle.load(file)" -) -citation = "bibtex\n@inproceedings{...,year={2020}}" - model_card = card.Card(model) @@ -99,12 +87,20 @@ model=pkl_name, requirements=[f"scikit-learn={sklearn.__version__}"], dst=local_repo ) -model_card.save( - os.path.join(f"{local_repo}", "README.md"), - card_data=card_data, - limitations=limitations, - model_description=model_description, - citation_bibtex=citation, - model_card_authors=model_card_authors, - get_started_code=get_started_code, +limitations = "This model is not ready to be used in production." +model_description = ( + "This is a HistGradientBoostingClassifier model trained on breast cancer dataset." + " It's trained with Halving Grid Search Cross Validation, with parameter grids on" + " max_leaf_nodes and max_depth." +) +model_card_authors = "skops_user" +get_started_code = ( + "import pickle\nwith open(dtc_pkl_filename, 'rb') as file:\nclf = pickle.load(file)" +) + +model_card.add("citation", "bibtex\n@inproceedings{...,year={2020}}") +model_card.add("get_started_code", get_started_code).add( + "model_card_authors", model_card_authors ) +model_card.add("limitations", limitations).add("model_description", model_description) +model_card.save(os.path.join(f"{local_repo}", "README.md"), card_data=card_data) diff --git a/skops/card/_model_card.py b/skops/card/_model_card.py index 30f5f676..f65ba980 100644 --- a/skops/card/_model_card.py +++ b/skops/card/_model_card.py @@ -12,35 +12,42 @@ def __init__(self, model): self.model = model self.hyperparameters_table = self.extract_estimator_config(self.model) self.model_plot = re.sub(r"\n\s+", "", str(estimator_html_repr(model))) + self.template_sections = {} - def add(self): - pass + def add(self, section, value): + """Takes values to fill model card template. + Parameters: + ---------- + section: str + Section in the template. + value: str + Value to fill in the section. + """ + self.template_sections[section] = value + return self def add_inspection(self): pass - def save(self, path, card_data, **card_kwargs): + def save(self, path, card_data): """Fills model card template, renders and saves it as markdown file to the target directory. Parameters: ---------- card_data: CardData CardData object. - card_kwargs: - Card kwargs are information you can pass to fill in the sections of the - card template, e.g. model_description, citation_bibtex, get_started_code. """ ROOT = skops.__path__ card_data.library_name = "sklearn" - template_path = card_kwargs.get("template_path") + template_path = self.template_sections.get("template_path") if template_path is None: template_path = os.path.join(f"{ROOT[0]}", "card", "default_template.md") - card_kwargs["template_path"] = template_path + self.template_sections["template_path"] = template_path card = ModelCard.from_template( card_data=card_data, hyperparameter_table=self.hyperparameter_table, model_plot=self.model_plot, - **card_kwargs, + **self.template_sections, ) card.save(path) From c9414b4856369e9d6345dcdfb54cb69b4fe7c2e3 Mon Sep 17 00:00:00 2001 From: merveenoyan Date: Wed, 20 Jul 2022 12:19:00 +0200 Subject: [PATCH 04/62] added plots & more --- examples/plot_hf_hub.py | 2 +- examples/plot_model_card.py | 25 +++++++----- skops/card/__init__.py | 4 +- skops/card/_model_card.py | 72 ++++++++++++++++++++++++++++------ skops/card/default_template.md | 3 +- skops/card/tests/test_card.py | 18 ++------- 6 files changed, 84 insertions(+), 40 deletions(-) diff --git a/examples/plot_hf_hub.py b/examples/plot_hf_hub.py index d1fb4f4c..26b8f0b5 100644 --- a/examples/plot_hf_hub.py +++ b/examples/plot_hf_hub.py @@ -87,7 +87,7 @@ # Model Card # ========== card_data = CardData(tags=["tabular-classification"]) -model_card = card.create_model_card(model, card_data) +model_card = card.Card(model) model_card.save(os.path.join(f"{local_repo}", "README.md")) # %% diff --git a/examples/plot_model_card.py b/examples/plot_model_card.py index 54dbebff..e5e84b6f 100644 --- a/examples/plot_model_card.py +++ b/examples/plot_model_card.py @@ -15,8 +15,8 @@ import pickle from tempfile import mkdtemp, mkstemp +import matplotlib.pyplot as plt import sklearn -from modelcards import CardData from sklearn.datasets import load_breast_cancer from sklearn.ensemble import HistGradientBoostingClassifier from sklearn.experimental import enable_halving_search_cv # noqa @@ -66,13 +66,6 @@ license = "mit" -card_data = CardData( - license=license, - tags=["tabular-classification"], - datasets="breast-cancer", - metrics=["acc"], -) - model_card = card.Card(model) @@ -103,4 +96,18 @@ "model_card_authors", model_card_authors ) model_card.add("limitations", limitations).add("model_description", model_description) -model_card.save(os.path.join(f"{local_repo}", "README.md"), card_data=card_data) + +plt.plot([1, 2, 3, 4]) +plt.ylabel("some numbers") +plt.savefig(f"{local_repo}/fig1.png") + +plt.plot([4, 5, 6, 7]) +plt.ylabel("some numbers") +plt.savefig(f"{local_repo}/fig2.png") + + +model_card.add_inspection("fig1", "fig1.png") +model_card.add_inspection("fig2", "fig2.png") + + +model_card.save(os.path.join(f"{local_repo}", "README.md")) diff --git a/skops/card/__init__.py b/skops/card/__init__.py index df78d1e7..bcdc27a7 100644 --- a/skops/card/__init__.py +++ b/skops/card/__init__.py @@ -1,3 +1,3 @@ -from ._model_card import create_model_card +from ._model_card import Card -__all__ = ["create_model_card"] +__all__ = ["Card"] diff --git a/skops/card/_model_card.py b/skops/card/_model_card.py index f65ba980..36a279e8 100644 --- a/skops/card/_model_card.py +++ b/skops/card/_model_card.py @@ -1,7 +1,7 @@ import os import re -from modelcards import ModelCard +from modelcards import CardData, ModelCard from sklearn.utils import estimator_html_repr import skops @@ -10,9 +10,10 @@ class Card: def __init__(self, model): self.model = model - self.hyperparameters_table = self.extract_estimator_config(self.model) + self.hyperparameter_table = self.extract_estimator_config() self.model_plot = re.sub(r"\n\s+", "", str(estimator_html_repr(model))) self.template_sections = {} + self.figure_paths = {} def add(self, section, value): """Takes values to fill model card template. @@ -26,23 +27,68 @@ def add(self, section, value): self.template_sections[section] = value return self - def add_inspection(self): - pass + def add_inspection(self, plot_name, plot_path): + """Takes plots to fill model card template. + Parameters: + ---------- + plot_path: str + Plot path as a string. + plot_name: str + Name of the plot. + """ + self.template_sections[plot_name] = plot_path + return self - def save(self, path, card_data): + def save(self, path): """Fills model card template, renders and saves it as markdown file to the target directory. Parameters: ---------- - card_data: CardData - CardData object. + path: Path + filepath to save your card. """ ROOT = skops.__path__ + + metadata_keys = [ + "language", + "license", + "library_name", + "tags", + "datasets", + "model_name", + "metrics", + "model-index", + ] + card_data_keys = {} + + # if key is supposed to be in metadata and is provided by user, write it to card_data_keys + for key in metadata_keys: + if key in self.template_sections.keys(): + card_data_keys[key] = self.template_sections.pop(key, "") + + # construct CardData + card_data = CardData(**card_data_keys) card_data.library_name = "sklearn" - template_path = self.template_sections.get("template_path") - if template_path is None: - template_path = os.path.join(f"{ROOT[0]}", "card", "default_template.md") - self.template_sections["template_path"] = template_path + + if self.template_sections.get("template_path") is None: + self.template_sections["template_path"] = os.path.join( + f"{ROOT[0]}", "card", "default_template.md" + ) + + # append plot_name if any plots are provided, at the end of the template + template = open(self.template_sections["template_path"], "a") + + with open(self.template_sections["template_path"], "r") as f: + template_content = f.read() + breakpoint() + for section in self.template_sections: + if section not in template_content and section != "template_path": + template.write( + f"\n\n{section}\n" + + f"![{section}]({self.template_sections[section]})\n\n" + ) + template.close() + card = ModelCard.from_template( card_data=card_data, hyperparameter_table=self.hyperparameter_table, @@ -51,7 +97,7 @@ def save(self, path, card_data): ) card.save(path) - def extract_estimator_config(model): + def extract_estimator_config(self): """Extracts estimator configuration and renders them into a vertical table. Parameters @@ -64,7 +110,7 @@ def extract_estimator_config(model): Markdown table of hyperparameters. """ - hyperparameter_dict = model.get_params(deep=True) + hyperparameter_dict = self.model.get_params(deep=True) table = "| Hyperparameters | Value |\n| :-- | :-- |\n" for hyperparameter, value in hyperparameter_dict.items(): table += f"| {hyperparameter} | {value} |\n" diff --git a/skops/card/default_template.md b/skops/card/default_template.md index 4664373f..65275c1f 100644 --- a/skops/card/default_template.md +++ b/skops/card/default_template.md @@ -62,5 +62,6 @@ You can contact the model card authors through following channels: Below you can find information related to citation. **BibTeX:** - +``` {{ citation_bibtex | default("[More Information Needed]", true)}} +``` diff --git a/skops/card/tests/test_card.py b/skops/card/tests/test_card.py index a5fb2fe9..5ad62a27 100644 --- a/skops/card/tests/test_card.py +++ b/skops/card/tests/test_card.py @@ -2,10 +2,9 @@ import tempfile import numpy as np -from modelcards import CardData from sklearn.linear_model import LinearRegression -from skops.card import create_model_card +from skops.card import Card def fit_model(): @@ -17,24 +16,15 @@ def fit_model(): def generate_card(): model = fit_model() - card_data = CardData(library_name="sklearn") - - model_card = create_model_card( - model, - card_data, - template_path="skops/card/default_template.md", - model_description="sklearn FTW", - ) + + model_card = Card(model) return model_card def test_write_model_card(): with tempfile.TemporaryDirectory(prefix="skops-test") as dir_path: model = fit_model() - card_data = CardData(library_name="sklearn") - model_card = create_model_card( - model, card_data=card_data, model_description="sklearn FTW" - ) + model_card = Card(model) model_card.save(os.path.join(f"{dir_path}", "README.md")) with open(os.path.join(f"{dir_path}", "README.md"), "r") as f: model_card = f.read() From 60002946f7fb0b9cedec90863dc2326702cab4e9 Mon Sep 17 00:00:00 2001 From: merveenoyan Date: Wed, 20 Jul 2022 13:15:00 +0200 Subject: [PATCH 05/62] addressed comments --- skops/card/_model_card.py | 20 ++++++++------------ skops/card/default_template.md | 9 +++++++++ 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/skops/card/_model_card.py b/skops/card/_model_card.py index 36a279e8..b89a88f8 100644 --- a/skops/card/_model_card.py +++ b/skops/card/_model_card.py @@ -10,7 +10,8 @@ class Card: def __init__(self, model): self.model = model - self.hyperparameter_table = self.extract_estimator_config() + self.hyperparameter_table = self._extract_estimator_config() + # the spaces in the pipeline breaks markdown, so we replace them self.model_plot = re.sub(r"\n\s+", "", str(estimator_html_repr(model))) self.template_sections = {} self.figure_paths = {} @@ -36,7 +37,7 @@ def add_inspection(self, plot_name, plot_path): plot_name: str Name of the plot. """ - self.template_sections[plot_name] = plot_path + self.figure_paths[plot_name] = plot_path return self def save(self, path): @@ -78,15 +79,10 @@ def save(self, path): # append plot_name if any plots are provided, at the end of the template template = open(self.template_sections["template_path"], "a") - with open(self.template_sections["template_path"], "r") as f: - template_content = f.read() - breakpoint() - for section in self.template_sections: - if section not in template_content and section != "template_path": - template.write( - f"\n\n{section}\n" - + f"![{section}]({self.template_sections[section]})\n\n" - ) + for plot in self.figure_paths: + template.write( + f"\n\n{plot}\n" + f"![{plot}]({self.figure_paths[plot]})\n\n" + ) template.close() card = ModelCard.from_template( @@ -97,7 +93,7 @@ def save(self, path): ) card.save(path) - def extract_estimator_config(self): + def _extract_estimator_config(self): """Extracts estimator configuration and renders them into a vertical table. Parameters diff --git a/skops/card/default_template.md b/skops/card/default_template.md index 65275c1f..c5d5a03e 100644 --- a/skops/card/default_template.md +++ b/skops/card/default_template.md @@ -65,3 +65,12 @@ Below you can find information related to citation. ``` {{ citation_bibtex | default("[More Information Needed]", true)}} ``` + + +fig1 +![fig1](fig1.png) + + + +fig2 +![fig2](fig2.png) From c9331ae7163a4658bc9070cb6b6f2431c207d666 Mon Sep 17 00:00:00 2001 From: Merve Noyan Date: Wed, 20 Jul 2022 13:15:21 +0200 Subject: [PATCH 06/62] Update skops/card/_model_card.py Co-authored-by: Adrin Jalali --- skops/card/_model_card.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/skops/card/_model_card.py b/skops/card/_model_card.py index b89a88f8..d760b0d2 100644 --- a/skops/card/_model_card.py +++ b/skops/card/_model_card.py @@ -16,16 +16,17 @@ def __init__(self, model): self.template_sections = {} self.figure_paths = {} - def add(self, section, value): + def add(self, **kwargs): """Takes values to fill model card template. Parameters: ---------- - section: str - Section in the template. - value: str - Value to fill in the section. + **kwargs : dict + Parameters to be set for the model card. These parameters + need to be sections of the underlying `jinja` template used. + """ - self.template_sections[section] = value + for section, value in kwargs.items(): + self.template_sections[section] = value return self def add_inspection(self, plot_name, plot_path): From 4901ee07cf92b9cdce33c9f3dee20a49ba484f1d Mon Sep 17 00:00:00 2001 From: merveenoyan Date: Wed, 20 Jul 2022 16:53:46 +0200 Subject: [PATCH 07/62] added docs and fixed tests --- examples/plot_hf_hub.py | 3 +-- examples/plot_model_card.py | 49 +++++++++++++++++++++------------- skops/card/_model_card.py | 11 ++++---- skops/card/default_template.md | 9 ------- skops/card/tests/test_card.py | 33 ++++++++++++++++++----- 5 files changed, 63 insertions(+), 42 deletions(-) diff --git a/examples/plot_hf_hub.py b/examples/plot_hf_hub.py index 26b8f0b5..d5fd4bb1 100644 --- a/examples/plot_hf_hub.py +++ b/examples/plot_hf_hub.py @@ -20,7 +20,6 @@ import sklearn from huggingface_hub import HfApi -from modelcards import CardData from sklearn.datasets import load_breast_cancer from sklearn.ensemble import HistGradientBoostingClassifier from sklearn.experimental import enable_halving_search_cv # noqa @@ -86,7 +85,7 @@ # %% # Model Card # ========== -card_data = CardData(tags=["tabular-classification"]) +# We will now create a model card and save it model_card = card.Card(model) model_card.save(os.path.join(f"{local_repo}", "README.md")) diff --git a/examples/plot_model_card.py b/examples/plot_model_card.py index e5e84b6f..5b28cef6 100644 --- a/examples/plot_model_card.py +++ b/examples/plot_model_card.py @@ -20,6 +20,7 @@ from sklearn.datasets import load_breast_cancer from sklearn.ensemble import HistGradientBoostingClassifier from sklearn.experimental import enable_halving_search_cv # noqa +from sklearn.metrics import ConfusionMatrixDisplay, confusion_matrix from sklearn.model_selection import HalvingGridSearchCV, train_test_split from skops import card, hub_utils @@ -60,26 +61,36 @@ # Create a model card # ==================== # We now create a model card, set couple of attributes and save it. -# We first set the metadata with CardData and pass it to create_model_card. -# Then, we pass information other than metadata in kwargs. +# Then, we pass information using add() and plots using add_inspection() # We'll initialize a local repository and save the card with the model in it. -license = "mit" - - model_card = card.Card(model) +# %% +# Initialize a repository to save our files in +# ==================== +# We will now initialize a repository and save our model _, pkl_name = mkstemp(prefix="skops-", suffix=".pkl") with open(pkl_name, mode="bw") as f: pickle.dump(model, file=f) local_repo = mkdtemp(prefix="skops-") + hub_utils.init( model=pkl_name, requirements=[f"scikit-learn={sklearn.__version__}"], dst=local_repo ) +# %% +# Pass information and plots to our model card +# ==================== +# We will pass information to fill our model card +# We will add plots to our card, note that these plots don't necessarily +# have to have a section in our template +# We will save the plots, and then pass plot name with path to add_inspection + +license = "mit" limitations = "This model is not ready to be used in production." model_description = ( "This is a HistGradientBoostingClassifier model trained on breast cancer dataset." @@ -91,23 +102,23 @@ "import pickle\nwith open(dtc_pkl_filename, 'rb') as file:\nclf = pickle.load(file)" ) -model_card.add("citation", "bibtex\n@inproceedings{...,year={2020}}") -model_card.add("get_started_code", get_started_code).add( - "model_card_authors", model_card_authors +model_card.add(citation="bibtex\n@inproceedings{...,year={2020}}") +model_card.add(get_started_code=get_started_code).add( + model_card_authors=model_card_authors ) -model_card.add("limitations", limitations).add("model_description", model_description) - -plt.plot([1, 2, 3, 4]) -plt.ylabel("some numbers") -plt.savefig(f"{local_repo}/fig1.png") +model_card.add("limitations", limitations).add(model_description=model_description) +predictions = model.predict(X_test) +cm = confusion_matrix(y_test, predictions, labels=model.classes_) +disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=model.classes_) +disp.plot() -plt.plot([4, 5, 6, 7]) -plt.ylabel("some numbers") -plt.savefig(f"{local_repo}/fig2.png") +plt.savefig(f"{local_repo}/confusion_matrix.png") +model_card.add_inspection(confusion_matrix="confusion_matrix.png") -model_card.add_inspection("fig1", "fig1.png") -model_card.add_inspection("fig2", "fig2.png") - +# %% +# Save model card +# ==================== +# We can simply save our model card by providing a path to save() model_card.save(os.path.join(f"{local_repo}", "README.md")) diff --git a/skops/card/_model_card.py b/skops/card/_model_card.py index d760b0d2..c0f8cf7a 100644 --- a/skops/card/_model_card.py +++ b/skops/card/_model_card.py @@ -29,16 +29,15 @@ def add(self, **kwargs): self.template_sections[section] = value return self - def add_inspection(self, plot_name, plot_path): + def add_inspection(self, **kwargs): """Takes plots to fill model card template. Parameters: ---------- - plot_path: str - Plot path as a string. - plot_name: str - Name of the plot. + Parameters to be set for the model card. These parameters + need to be sections of the underlying `jinja` template used. """ - self.figure_paths[plot_name] = plot_path + for plot_name, plot_path in kwargs.items(): + self.figure_paths[plot_name] = plot_path return self def save(self, path): diff --git a/skops/card/default_template.md b/skops/card/default_template.md index c5d5a03e..65275c1f 100644 --- a/skops/card/default_template.md +++ b/skops/card/default_template.md @@ -65,12 +65,3 @@ Below you can find information related to citation. ``` {{ citation_bibtex | default("[More Information Needed]", true)}} ``` - - -fig1 -![fig1](fig1.png) - - - -fig2 -![fig2](fig2.png) diff --git a/skops/card/tests/test_card.py b/skops/card/tests/test_card.py index 5ad62a27..49d428ed 100644 --- a/skops/card/tests/test_card.py +++ b/skops/card/tests/test_card.py @@ -1,6 +1,7 @@ import os import tempfile +import matplotlib.pyplot as plt import numpy as np from sklearn.linear_model import LinearRegression @@ -16,19 +17,16 @@ def fit_model(): def generate_card(): model = fit_model() - model_card = Card(model) return model_card -def test_write_model_card(): +def test_save_model_card(): with tempfile.TemporaryDirectory(prefix="skops-test") as dir_path: model = fit_model() model_card = Card(model) model_card.save(os.path.join(f"{dir_path}", "README.md")) - with open(os.path.join(f"{dir_path}", "README.md"), "r") as f: - model_card = f.read() - assert "sklearn FTW" in model_card + assert os.path.exists(f"{dir_path}/README.md") def test_hyperparameter_table(): @@ -46,4 +44,27 @@ def test_plot_model(): model_card.save(os.path.join(f"{dir_path}", "README.md")) with open(os.path.join(f"{dir_path}", "README.md"), "r") as f: model_card = f.read() - assert "
LinearRegression()
Please rerun this cell to show the HTML repr or trust the notebook.
+ +# How to Get Started with the Model + +Use the code below to get started with the model. + +
+ Click to expand + +``` +[More Information Needed] + +``` + +
+ + + + +# Model Card Authors + +This model card is written by following authors: + +[More Information Needed] + +# Model Card Contact + +You can contact the model card authors through following channels: +[More Information Needed] + +# Citation + +Below you can find information related to citation. + +**BibTeX:** +``` +[More Information Needed] +``` diff --git a//README.md b//README.md new file mode 100644 index 00000000..1f1afd96 --- /dev/null +++ b//README.md @@ -0,0 +1,74 @@ +--- +library_name: sklearn +--- + +# Model description + +[More Information Needed] + +## Intended uses & limitations + +[More Information Needed] + +## Training Procedure + +### Hyperparameters + +The model is trained with below hyperparameters. + +
+ Click to expand + +| Hyperparameters | Value | +| :-- | :-- | +| copy_X | True | +| fit_intercept | True | +| n_jobs | None | +| normalize | deprecated | +| positive | False | + + +
+ +### Model Plot + +The model plot is below. + +
LinearRegression()
Please rerun this cell to show the HTML repr or trust the notebook.
+ +# How to Get Started with the Model + +Use the code below to get started with the model. + +
+ Click to expand + +``` +[More Information Needed] + +``` + +
+ + + + +# Model Card Authors + +This model card is written by following authors: + +[More Information Needed] + +# Model Card Contact + +You can contact the model card authors through following channels: +[More Information Needed] + +# Citation + +Below you can find information related to citation. + +**BibTeX:** +``` +[More Information Needed] +``` diff --git a//README.md b//README.md new file mode 100644 index 00000000..6f1cb69e --- /dev/null +++ b//README.md @@ -0,0 +1,74 @@ +--- +library_name: sklearn +--- + +# Model description + +[More Information Needed] + +## Intended uses & limitations + +[More Information Needed] + +## Training Procedure + +### Hyperparameters + +The model is trained with below hyperparameters. + +
+ Click to expand + +| Hyperparameters | Value | +| :-- | :-- | +| copy_X | True | +| fit_intercept | True | +| n_jobs | None | +| normalize | deprecated | +| positive | False | + + +
+ +### Model Plot + +The model plot is below. + +
LinearRegression()
Please rerun this cell to show the HTML repr or trust the notebook.
+ +# How to Get Started with the Model + +Use the code below to get started with the model. + +
+ Click to expand + +``` +[More Information Needed] + +``` + +
+ + + + +# Model Card Authors + +This model card is written by following authors: + +[More Information Needed] + +# Model Card Contact + +You can contact the model card authors through following channels: +[More Information Needed] + +# Citation + +Below you can find information related to citation. + +**BibTeX:** +``` +[More Information Needed] +``` diff --git a//README.md b//README.md new file mode 100644 index 00000000..a07ce4ae --- /dev/null +++ b//README.md @@ -0,0 +1,74 @@ +--- +library_name: sklearn +--- + +# Model description + +[More Information Needed] + +## Intended uses & limitations + +[More Information Needed] + +## Training Procedure + +### Hyperparameters + +The model is trained with below hyperparameters. + +
+ Click to expand + +| Hyperparameters | Value | +| :-- | :-- | +| copy_X | True | +| fit_intercept | True | +| n_jobs | None | +| normalize | deprecated | +| positive | False | + + +
+ +### Model Plot + +The model plot is below. + +
LinearRegression()
Please rerun this cell to show the HTML repr or trust the notebook.
+ +# How to Get Started with the Model + +Use the code below to get started with the model. + +
+ Click to expand + +``` +[More Information Needed] + +``` + +
+ + + + +# Model Card Authors + +This model card is written by following authors: + +[More Information Needed] + +# Model Card Contact + +You can contact the model card authors through following channels: +[More Information Needed] + +# Citation + +Below you can find information related to citation. + +**BibTeX:** +``` +[More Information Needed] +``` diff --git a//README.md b//README.md new file mode 100644 index 00000000..3236710b --- /dev/null +++ b//README.md @@ -0,0 +1,74 @@ +--- +library_name: sklearn +--- + +# Model description + +[More Information Needed] + +## Intended uses & limitations + +[More Information Needed] + +## Training Procedure + +### Hyperparameters + +The model is trained with below hyperparameters. + +
+ Click to expand + +| Hyperparameters | Value | +| :-- | :-- | +| copy_X | True | +| fit_intercept | True | +| n_jobs | None | +| normalize | deprecated | +| positive | False | + + +
+ +### Model Plot + +The model plot is below. + +
LinearRegression()
Please rerun this cell to show the HTML repr or trust the notebook.
+ +# How to Get Started with the Model + +Use the code below to get started with the model. + +
+ Click to expand + +``` +[More Information Needed] + +``` + +
+ + + + +# Model Card Authors + +This model card is written by following authors: + +[More Information Needed] + +# Model Card Contact + +You can contact the model card authors through following channels: +[More Information Needed] + +# Citation + +Below you can find information related to citation. + +**BibTeX:** +``` +[More Information Needed] +``` diff --git a//README.md b//README.md new file mode 100644 index 00000000..6e5572d3 --- /dev/null +++ b//README.md @@ -0,0 +1,74 @@ +--- +library_name: sklearn +--- + +# Model description + +[More Information Needed] + +## Intended uses & limitations + +[More Information Needed] + +## Training Procedure + +### Hyperparameters + +The model is trained with below hyperparameters. + +
+ Click to expand + +| Hyperparameters | Value | +| :-- | :-- | +| copy_X | True | +| fit_intercept | True | +| n_jobs | None | +| normalize | deprecated | +| positive | False | + + +
+ +### Model Plot + +The model plot is below. + +
LinearRegression()
Please rerun this cell to show the HTML repr or trust the notebook.
+ +# How to Get Started with the Model + +Use the code below to get started with the model. + +
+ Click to expand + +``` +[More Information Needed] + +``` + +
+ + + + +# Model Card Authors + +This model card is written by following authors: + +[More Information Needed] + +# Model Card Contact + +You can contact the model card authors through following channels: +[More Information Needed] + +# Citation + +Below you can find information related to citation. + +**BibTeX:** +``` +[More Information Needed] +``` diff --git a/skops/card/tests/test_card.py b/skops/card/tests/test_card.py index f55301b8..b6724141 100644 --- a/skops/card/tests/test_card.py +++ b/skops/card/tests/test_card.py @@ -17,10 +17,11 @@ def fit_model(): return reg -def generate_card(model_diagram=True): +@pytest.fixture +def model_card(model_diagram=True): model = fit_model() - model_card = Card(model, model_diagram) - return model_card + card = Card(model, model_diagram) + yield card @pytest.fixture @@ -29,39 +30,35 @@ def destination_path(): yield Path(dir_path) -def test_save_model_card(destination_path): - model = fit_model() - model_card = Card(model) +def test_save_model_card(destination_path, model_card): model_card.save((Path(destination_path) / "README.md")) assert (Path(destination_path) / "README.md").exists() -def test_hyperparameter_table(destination_path): - model_card = generate_card() +def test_hyperparameter_table(destination_path, model_card): model_card.save((Path(destination_path) / "README.md")) with open((Path(destination_path) / "README.md"), "r") as f: model_card = f.read() assert "fit_intercept" in model_card -def test_plot_model(destination_path): - model_card = generate_card() +def test_plot_model(destination_path, model_card): model_card.save((Path(destination_path) / "README.md")) with open((Path(destination_path) / "README.md"), "r") as f: model_card = f.read() assert "
LinearRegression()
Please rerun this cell to show the HTML repr or trust the notebook.
- -# How to Get Started with the Model - -Use the code below to get started with the model. - -
- Click to expand - -``` -[More Information Needed] - -``` - -
- - - - -# Model Card Authors - -This model card is written by following authors: - -[More Information Needed] - -# Model Card Contact - -You can contact the model card authors through following channels: -[More Information Needed] - -# Citation - -Below you can find information related to citation. - -**BibTeX:** -``` -[More Information Needed] -``` diff --git a//README.md b//README.md deleted file mode 100644 index 1f1afd96..00000000 --- a//README.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -library_name: sklearn ---- - -# Model description - -[More Information Needed] - -## Intended uses & limitations - -[More Information Needed] - -## Training Procedure - -### Hyperparameters - -The model is trained with below hyperparameters. - -
- Click to expand - -| Hyperparameters | Value | -| :-- | :-- | -| copy_X | True | -| fit_intercept | True | -| n_jobs | None | -| normalize | deprecated | -| positive | False | - - -
- -### Model Plot - -The model plot is below. - -
LinearRegression()
Please rerun this cell to show the HTML repr or trust the notebook.
- -# How to Get Started with the Model - -Use the code below to get started with the model. - -
- Click to expand - -``` -[More Information Needed] - -``` - -
- - - - -# Model Card Authors - -This model card is written by following authors: - -[More Information Needed] - -# Model Card Contact - -You can contact the model card authors through following channels: -[More Information Needed] - -# Citation - -Below you can find information related to citation. - -**BibTeX:** -``` -[More Information Needed] -``` diff --git a//README.md b//README.md deleted file mode 100644 index 6f1cb69e..00000000 --- a//README.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -library_name: sklearn ---- - -# Model description - -[More Information Needed] - -## Intended uses & limitations - -[More Information Needed] - -## Training Procedure - -### Hyperparameters - -The model is trained with below hyperparameters. - -
- Click to expand - -| Hyperparameters | Value | -| :-- | :-- | -| copy_X | True | -| fit_intercept | True | -| n_jobs | None | -| normalize | deprecated | -| positive | False | - - -
- -### Model Plot - -The model plot is below. - -
LinearRegression()
Please rerun this cell to show the HTML repr or trust the notebook.
- -# How to Get Started with the Model - -Use the code below to get started with the model. - -
- Click to expand - -``` -[More Information Needed] - -``` - -
- - - - -# Model Card Authors - -This model card is written by following authors: - -[More Information Needed] - -# Model Card Contact - -You can contact the model card authors through following channels: -[More Information Needed] - -# Citation - -Below you can find information related to citation. - -**BibTeX:** -``` -[More Information Needed] -``` diff --git a//README.md b//README.md deleted file mode 100644 index a07ce4ae..00000000 --- a//README.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -library_name: sklearn ---- - -# Model description - -[More Information Needed] - -## Intended uses & limitations - -[More Information Needed] - -## Training Procedure - -### Hyperparameters - -The model is trained with below hyperparameters. - -
- Click to expand - -| Hyperparameters | Value | -| :-- | :-- | -| copy_X | True | -| fit_intercept | True | -| n_jobs | None | -| normalize | deprecated | -| positive | False | - - -
- -### Model Plot - -The model plot is below. - -
LinearRegression()
Please rerun this cell to show the HTML repr or trust the notebook.
- -# How to Get Started with the Model - -Use the code below to get started with the model. - -
- Click to expand - -``` -[More Information Needed] - -``` - -
- - - - -# Model Card Authors - -This model card is written by following authors: - -[More Information Needed] - -# Model Card Contact - -You can contact the model card authors through following channels: -[More Information Needed] - -# Citation - -Below you can find information related to citation. - -**BibTeX:** -``` -[More Information Needed] -``` diff --git a//README.md b//README.md deleted file mode 100644 index 3236710b..00000000 --- a//README.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -library_name: sklearn ---- - -# Model description - -[More Information Needed] - -## Intended uses & limitations - -[More Information Needed] - -## Training Procedure - -### Hyperparameters - -The model is trained with below hyperparameters. - -
- Click to expand - -| Hyperparameters | Value | -| :-- | :-- | -| copy_X | True | -| fit_intercept | True | -| n_jobs | None | -| normalize | deprecated | -| positive | False | - - -
- -### Model Plot - -The model plot is below. - -
LinearRegression()
Please rerun this cell to show the HTML repr or trust the notebook.
- -# How to Get Started with the Model - -Use the code below to get started with the model. - -
- Click to expand - -``` -[More Information Needed] - -``` - -
- - - - -# Model Card Authors - -This model card is written by following authors: - -[More Information Needed] - -# Model Card Contact - -You can contact the model card authors through following channels: -[More Information Needed] - -# Citation - -Below you can find information related to citation. - -**BibTeX:** -``` -[More Information Needed] -``` diff --git a//README.md b//README.md deleted file mode 100644 index 6e5572d3..00000000 --- a//README.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -library_name: sklearn ---- - -# Model description - -[More Information Needed] - -## Intended uses & limitations - -[More Information Needed] - -## Training Procedure - -### Hyperparameters - -The model is trained with below hyperparameters. - -
- Click to expand - -| Hyperparameters | Value | -| :-- | :-- | -| copy_X | True | -| fit_intercept | True | -| n_jobs | None | -| normalize | deprecated | -| positive | False | - - -
- -### Model Plot - -The model plot is below. - -
LinearRegression()
Please rerun this cell to show the HTML repr or trust the notebook.
- -# How to Get Started with the Model - -Use the code below to get started with the model. - -
- Click to expand - -``` -[More Information Needed] - -``` - -
- - - - -# Model Card Authors - -This model card is written by following authors: - -[More Information Needed] - -# Model Card Contact - -You can contact the model card authors through following channels: -[More Information Needed] - -# Citation - -Below you can find information related to citation. - -**BibTeX:** -``` -[More Information Needed] -``` From bfc9f4d43a458f57a8091567865fc7e5f40bae3d Mon Sep 17 00:00:00 2001 From: merveenoyan Date: Tue, 26 Jul 2022 16:01:37 +0200 Subject: [PATCH 57/62] misc nits --- examples/plot_model_card.py | 9 ++++----- skops/card/_model_card.py | 12 +++++++++--- skops/card/tests/test_card.py | 2 +- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/examples/plot_model_card.py b/examples/plot_model_card.py index fe184f0f..6f4a3584 100644 --- a/examples/plot_model_card.py +++ b/examples/plot_model_card.py @@ -15,7 +15,6 @@ from pathlib import Path from tempfile import mkdtemp, mkstemp -import matplotlib.pyplot as plt import sklearn from sklearn.datasets import load_breast_cancer from sklearn.ensemble import HistGradientBoostingClassifier @@ -102,9 +101,9 @@ get_started_code = ( "import pickle\nwith open(dtc_pkl_filename, 'rb') as file:\nclf = pickle.load(file)" ) - +citation_bibtex = "bibtex\n@inproceedings{...,year={2020}}" model_card.add( - citation_bibtex="bibtex\n@inproceedings{...,year={2020}}", + citation_bibtex=citation_bibtex, get_started_code=get_started_code, model_card_authors=model_card_authors, limitations=limitations, @@ -115,9 +114,9 @@ disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=model.classes_) disp.plot() -plt.savefig(f"{local_repo}/confusion_matrix.png") +disp.figure_.savefig((Path(local_repo) / "confusion_matrix.png")) -model_card.add_plot(confusion_matrix="confusion_matrix.png") +model_card.add_plot(**{"confusion matrix": "confusion_matrix.png"}) # %% # Save model card diff --git a/skops/card/_model_card.py b/skops/card/_model_card.py index 63dd3e5d..0370df15 100644 --- a/skops/card/_model_card.py +++ b/skops/card/_model_card.py @@ -32,7 +32,6 @@ class Card: >>> from sklearn.metrics import ConfusionMatrixDisplay, confusion_matrix >>> from sklearn.datasets import load_iris >>> from sklearn.linear_model import LogisticRegression - >>> import matplotlib.pyplot as plt >>> from skops import card >>> X, y = load_iris(return_X_y=True) >>> model = LogisticRegression(random_state=0).fit(X, y) @@ -45,7 +44,7 @@ class Card: ... display_labels=model.classes_) >>> disp.plot() # doctest: +ELLIPSIS - >>> plt.savefig("confusion_matrix.png") + >>> disp.figure_.savefig("confusion_matrix.png") ... >>> model_card.add_plot(confusion_matrix="confusion_matrix.png") # doctest: +ELLIPSIS @@ -71,7 +70,10 @@ def add(self, **kwargs): **kwargs : dict Parameters to be set for the model card. These parameters need to be sections of the underlying `jinja` template used. - + Returns: + -------- + self : object + Card object. """ for section, value in kwargs.items(): self.template_sections[section] = value @@ -87,6 +89,10 @@ def add_plot(self, **kwargs): is the name of the plot and `plot_path` is the path to the plot, relative to the root of the project. The plots should have already been saved under the project's folder. + Returns: + -------- + self : object + Card object. """ for plot_name, plot_path in kwargs.items(): self._figure_paths[plot_name] = plot_path diff --git a/skops/card/tests/test_card.py b/skops/card/tests/test_card.py index b6724141..c59ba5e9 100644 --- a/skops/card/tests/test_card.py +++ b/skops/card/tests/test_card.py @@ -68,7 +68,7 @@ def test_add(destination_path, model_card): def test_add_plot(destination_path, model_card): plt.plot([4, 5, 6, 7]) - plt.savefig(f"{destination_path}/fig1.png") + plt.savefig((Path(destination_path) / "fig1.png")) model_card.add_plot(fig1="fig1.png") model_card.save(Path(destination_path) / "README.md") with open((Path(destination_path) / "README.md"), "r") as f: From adbb9e38d80c0b9d9c664d0de690bcd8e3731cd8 Mon Sep 17 00:00:00 2001 From: Merve Noyan Date: Tue, 26 Jul 2022 17:04:08 +0200 Subject: [PATCH 58/62] Update examples/plot_model_card.py Co-authored-by: Adrin Jalali --- examples/plot_model_card.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/plot_model_card.py b/examples/plot_model_card.py index 6f4a3584..c14db955 100644 --- a/examples/plot_model_card.py +++ b/examples/plot_model_card.py @@ -114,7 +114,7 @@ disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=model.classes_) disp.plot() -disp.figure_.savefig((Path(local_repo) / "confusion_matrix.png")) +disp.figure_.savefig(Path(local_repo) / "confusion_matrix.png") model_card.add_plot(**{"confusion matrix": "confusion_matrix.png"}) From e01143ddf2bcdce29c7b22ef9016d49698a34ed1 Mon Sep 17 00:00:00 2001 From: merveenoyan Date: Tue, 26 Jul 2022 17:11:33 +0200 Subject: [PATCH 59/62] removed extra parantheses --- examples/plot_hf_hub.py | 2 +- examples/plot_model_card.py | 4 ++-- skops/card/tests/test_card.py | 34 +++++++++++++++++----------------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/examples/plot_hf_hub.py b/examples/plot_hf_hub.py index 347e1871..819e539d 100644 --- a/examples/plot_hf_hub.py +++ b/examples/plot_hf_hub.py @@ -88,7 +88,7 @@ # ========== # We will now create a model card and save it model_card = card.Card(model) -model_card.save((Path(local_repo) / "README.md")) +model_card.save(Path(local_repo) / "README.md") # %% # Push to Hub diff --git a/examples/plot_model_card.py b/examples/plot_model_card.py index 6f4a3584..1d504f81 100644 --- a/examples/plot_model_card.py +++ b/examples/plot_model_card.py @@ -114,7 +114,7 @@ disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=model.classes_) disp.plot() -disp.figure_.savefig((Path(local_repo) / "confusion_matrix.png")) +disp.figure_.savefig(Path(local_repo) / "confusion_matrix.png") model_card.add_plot(**{"confusion matrix": "confusion_matrix.png"}) @@ -123,4 +123,4 @@ # =============== # We can simply save our model card by providing a path to ``save()`` -model_card.save((Path(local_repo) / "README.md")) +model_card.save(Path(local_repo) / "README.md") diff --git a/skops/card/tests/test_card.py b/skops/card/tests/test_card.py index c59ba5e9..439a4b33 100644 --- a/skops/card/tests/test_card.py +++ b/skops/card/tests/test_card.py @@ -31,20 +31,20 @@ def destination_path(): def test_save_model_card(destination_path, model_card): - model_card.save((Path(destination_path) / "README.md")) + model_card.save(Path(destination_path) / "README.md") assert (Path(destination_path) / "README.md").exists() def test_hyperparameter_table(destination_path, model_card): - model_card.save((Path(destination_path) / "README.md")) - with open((Path(destination_path) / "README.md"), "r") as f: + model_card.save(Path(destination_path) / "README.md") + with open(Path(destination_path) / "README.md", "r") as f: model_card = f.read() assert "fit_intercept" in model_card def test_plot_model(destination_path, model_card): - model_card.save((Path(destination_path) / "README.md")) - with open((Path(destination_path) / "README.md"), "r") as f: + model_card.save(Path(destination_path) / "README.md") + with open(Path(destination_path) / "README.md", "r") as f: model_card = f.read() assert "