V0.3.19

pyproject.toml

@@ -3,7 +3,7 @@ requires = ["setuptools>=61.0"]
 build-backend = "setuptools.build_meta"
 [project]
 name = "math-spec-mapping"
-version = "0.3.18"
+version = "0.3.19"
 authors = [
   { name="Sean McOwen", email="Sean@Block.Science" },
 ]

research_notes/2024-12-17.md

@@ -0,0 +1,85 @@
+# Report Creation Plan
+
+## Executive Summary
+
+- The report plan below details the current and desired future state of reporting in MSML
+- Feedback from users is encouraged as all plans are only loosely defined and meant to potentially be changed to be more useful
+
+## Current Reporting Architecture
+
+- There are a handful of files such as mechanisms.py, policies.py, etc. that have mostly depcreated functions from the HTML writer version.
+- All of the main abilities are housed in the markdown.py file that has each markdown file type broken down as functions, for example one for mechanisms, one for policies, etc.
+- All the writer functions create markdown files which when opened in Obsidian create a nicely linked documentation system.
+
+## Desired Future Reporting Architecture
+
+- [Switching to atomized functions](https://github.com/BlockScience/MSML/issues/165) will help to improve the ongoing maintenance of code
+- There is an aim to have a "plug and play" feel to the reporting modules that allow for flexibility and choosing exactly how you want reports
+    - Creating [style dictionaries](https://github.com/BlockScience/MSML/issues/251) is one way that reports can become flexible
+- There will be a desire to have multiple output avenues
+    - Standard markdown
+    - Markdown with wikilinks for use in Obsidian
+    - [PDFs of markdown](https://github.com/BlockScience/MSML/issues/600)
+- I want to also [create a mermaid graph](https://github.com/BlockScience/MSML/issues/601) or similar of the reporting architecture for both developer documentation as well as organizing thoughts on the best way to structure the codebase
+
+## Outputs
+
+1. Obsidian vault of markdown files (currently implemented)
+2. [Individual markdown reports](https://github.com/BlockScience/MSML/issues/602) of components based on style dictionary (for things such as how deep to go in terms of just saying what domain is versus actually listing out the domain objects such as {"name": "Space 1", "schema": {....}} for giving stakeholders the entire picture)
+    - And the option to PDF it
+3. [Specialized reports](https://github.com/BlockScience/MSML/issues/603) such as the parameter effects style whereby you can see every single block that is impacted downstream by parameters
+4. Potential for future engine extensions to support things such as stock & flow diagrams (although it can be shifted in terms of the priority)
+
+## Issue Log
+
+- The table below are all the open issues that can be prioritized in the upcoming milestones
+- I will plan to begin moving the issues to broad milestones such as V0.5, V0.6, V0.7, etc. to give rough estimates of when things will be implemented
+- There are quite a few improvement style issues which are good to also consider such as whether or not it is important that a user can add metrics and stateful metrics to the mermaid graphs
+
+|                 | Reporting                                                                                                                                                                 |
+|:----------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| High Priority   | [Create plan for specialized reports](https://github.com/BlockScience/MSML/issues/603)                                                                                    |
+|                 | [Create plan for individual markdown reports](https://github.com/BlockScience/MSML/issues/602)                                                                            |
+|                 | [Create a mermaid graph or similar of the reporting architecture](https://github.com/BlockScience/MSML/issues/601)                                                        |
+|                 | [Revive PDF Writing Abilities for Reporting](https://github.com/BlockScience/MSML/issues/600)                                                                             |
+|                 | [Add something for displaying source code for wiring](https://github.com/BlockScience/MSML/issues/570)                                                                    |
+|                 | [For policies/mechanisms/etc add in a section for markdown reporting that says wirings that it is involved in](https://github.com/BlockScience/MSML/issues/258)           |
+|                 | [Switch legacy reporting to be using mermaid + markdown](https://github.com/BlockScience/MSML/issues/155)                                                                 |
+|                 | [Add Table comparing policy options, boundary options, etc](https://github.com/BlockScience/MSML/issues/148)                                                              |
+| Medium Priority | [Obsidian Canvas Creator for Wirings](https://github.com/BlockScience/MSML/issues/541)                                                                                    |
+|                 | [Add Metrics and Stateful Metrics used to Mermaid Charts](https://github.com/BlockScience/MSML/issues/540)                                                                |
+|                 | [Stock and Flow generator](https://github.com/BlockScience/MSML/issues/498)                                                                                               |
+|                 | [Idea: Auto-displays](https://github.com/BlockScience/MSML/issues/280)                                                                                                    |
+|                 | [Convert action chain report to markdown](https://github.com/BlockScience/MSML/issues/254)                                                                                |
+|                 | [Convert entity report to markdown](https://github.com/BlockScience/MSML/issues/253)                                                                                      |
+|                 | [Make write basic report markdown](https://github.com/BlockScience/MSML/issues/252)                                                                                       |
+|                 | [Atomize Writing Functions](https://github.com/BlockScience/MSML/issues/165)                                                                                              |
+|                 | [Add Policy Report](https://github.com/BlockScience/MSML/issues/159)                                                                                                      |
+|                 | [Add parameters impacting to policy, behaviors, etc. in report write out](https://github.com/BlockScience/MSML/issues/89)                                                 |
+|                 | [Make a type of report that is a parameter report](https://github.com/BlockScience/MSML/issues/74)                                                                        |
+|                 | [Replicate Something like write_full_state_section from the alpha version](https://github.com/BlockScience/MSML/issues/58)                                                |
+| Low Priority    | [Add obsidian plugins to starter repos and have a function that also can populate them](https://github.com/BlockScience/MSML/issues/387)                                  |
+|                 | [Add ability to de-emphasize certain wirings by marking them with a flag that puts their reports into an "extra" wiring](https://github.com/BlockScience/MSML/issues/381) |
+|                 | [Figure out how to force CSS snippets used](https://github.com/BlockScience/MSML/issues/322)                                                                              |
+|                 | [Idea: Top level tag for wirings](https://github.com/BlockScience/MSML/issues/281)                                                                                        |
+|                 | [Add images assets functionality](https://github.com/BlockScience/MSML/issues/260)                                                                                        |
+|                 | [Mermaid graph assets functionality](https://github.com/BlockScience/MSML/issues/259)                                                                                     |
+|                 | [Style Dictionary options](https://github.com/BlockScience/MSML/issues/251)                                                                                               |
+|                 | [Consider a format dictionary for MSML that allows you to override certain formatting options](https://github.com/BlockScience/MSML/issues/246)                           |
+|                 | [Add metric linkages to the reporting for components](https://github.com/BlockScience/MSML/issues/233)                                                                    |
+|                 | [Metric report](https://github.com/BlockScience/MSML/issues/231)                                                                                                          |
+|                 | [Exclude Dictionary Feature](https://github.com/BlockScience/MSML/issues/79)                                                                                              |
+|                 | [Add Parameters to Graph Option](https://github.com/BlockScience/MSML/issues/53)                                                                                          |
+
+## Oustanding Questions
+
+1. How urgent is the style dictionary kind of customizability to people?
+2. Are there any other formats besides markdown and PDF that people are interested in having?
+3. What other kinds of reports would be desired?
+4. Are there any improvements to the current reporting that would be nice?
+
+## An Aside on Stock & Flow
+- I think stock & flow would add even more requirements on the JSON spec to have things such as a field for state variables used in the block
+- This, however, could also be kept as an optional field
+- And if code is bound to the component, then there might even be automation to actually read that data in automatically or create a specialized function for imputing it when you are feeling lazy but have the code bindings defined out already
+- It seems like a quite often used chart so I am curious to hear if others believe it is worth it to invest the time into this as it would also be a somewhat decent lift

src/math_spec_mapping/Classes/Block.py

@@ -172,6 +172,9 @@ def components_full(self):
         out = list(set(out))
         return out
 
+    def __repr__(self):
+        return "<{}>".format(self.name)
+
 
 class ParallelBlock(Block):
     def __init__(self, data: Dict):

src/math_spec_mapping/Classes/MathSpec.py

@@ -439,7 +439,14 @@ def _crawl_spaces(self):
 
         self._used_spaces = list(set().union(*self._used_spaces))
         us_names = [y.name for y in self._used_spaces]
-        self._unused_spaces = [self.spaces[x] for x in self.spaces if x not in us_names]
+        self._unused_spaces = [
+            self.spaces[x]
+            for x in self.spaces
+            if x not in us_names and x not in ["Terminating Space", "Empty Space"]
+        ]
+
+        if len(self._unused_spaces) > 0:
+            print("The following spaces are not used: {}".format(self._unused_spaces))
 
     def _add_spec_tree(self, tree):
         self.tree = tree
@@ -535,8 +542,17 @@ def _add_spec_tree(self, tree):
                             component
                         ]
             elif folder == "Displays":
-                print("Displays not implemented")
-                # keys = [x["name"] for x in ms.displays["Wiring"]]
+
+                for component in self.displays["Wiring"]:
+                    if component["name"] not in tree:
+                        print(
+                            "Can't find component code source in {} for {}".format(
+                                folder, component["name"]
+                            )
+                        )
+                        component["Source Code Location"] = None
+                    else:
+                        component["Source Code Location"] = tree[component["name"]]
             elif folder == "Spaces":
                 for component in self.spaces:
                     if component in ["Terminating Space", "Empty Space"]:
@@ -1010,6 +1026,14 @@ def load_control_actions(self):
                     )
                 else:
                     control_actions[ca.name] = opt.implementations["python"]
+
+                for opt_i in [x for x in ca.control_action_options if x != opt]:
+                    if "python" not in opt_i.implementations:
+                        print(
+                            "No python implementation for {} / {}. To fix this, go to Implementations/Python/ControlActions and add {}".format(
+                                ca.name, opt_i.name, opt_i.name
+                            )
+                        )
         return control_actions
 
     def load_boundary_actions(self):
@@ -1041,6 +1065,14 @@ def load_boundary_actions(self):
                     )
                 else:
                     boundary_actions[ba.name] = opt.implementations["python"]
+
+                for opt_i in [x for x in ba.boundary_action_options if x != opt]:
+                    if "python" not in opt_i.implementations:
+                        print(
+                            "No python implementation for {} / {}. To fix this, go to Implementations/Python/BoundaryActions and add {}".format(
+                                ba.name, opt_i.name, opt_i.name
+                            )
+                        )
         return boundary_actions
 
     def load_mechanisms(self):
@@ -1137,6 +1169,14 @@ def load_policies(self):
                     )
                 else:
                     policies[p.name] = opt.implementations["python"]
+            for opt_i in [x for x in p.policy_options if x != opt]:
+                if "python" not in opt_i.implementations:
+                    print(
+                        "No python implementation for {} / {}. To fix this, go to Implementations/Python/Policies and add {}".format(
+                            p.name, opt_i.name, opt_i.name
+                        )
+                    )
+
         return policies
 
     def load_stateful_metrics(self):

src/math_spec_mapping/Load/action_transmission_channel.py

@@ -50,8 +50,14 @@ def convert_action_transmission_channel(
         data["target"] = ms["Mechanisms"][target]
 
     # Add in called by and called here with origin and target
-    data["origin"].calls.append((data["target"], data["optional"], data["space"]))
-    data["target"].called_by.append((data["origin"], data["optional"], data["space"]))
+    if (data["target"], data["optional"], data["space"]) not in data["origin"].calls:
+        data["origin"].calls.append((data["target"], data["optional"], data["space"]))
+    if (data["origin"], data["optional"], data["space"]) not in data[
+        "target"
+    ].called_by:
+        data["target"].called_by.append(
+            (data["origin"], data["optional"], data["space"])
+        )
 
     # Build the action transmission channel object
     return ActionTransmissionChannel(data)

src/math_spec_mapping/Load/boundary_actions.py

@@ -1,6 +1,6 @@
 from typing import Dict
 from ..Classes import BoundaryAction, BoundaryActionOption
-from .general import check_json_keys
+from .general import check_json_keys, check_domain_codomain_spaces
 
 
 def convert_boundary_action(data: Dict, ms: Dict) -> BoundaryAction:
@@ -24,6 +24,8 @@ def convert_boundary_action(data: Dict, ms: Dict) -> BoundaryAction:
         data["name"]
     )
 
+    check_domain_codomain_spaces(data, ms)
+
     if len(data["codomain"]) == 0:
         data["codomain"] = ("Empty Space",)
 

src/math_spec_mapping/Load/control_actions.py

@@ -1,6 +1,6 @@
 from typing import Dict
 from ..Classes import ControlAction, ControlActionOption
-from .general import check_json_keys
+from .general import check_json_keys, check_domain_codomain_spaces
 
 
 def convert_control_action(data: Dict, ms: Dict) -> ControlAction:
@@ -25,6 +25,8 @@ def convert_control_action(data: Dict, ms: Dict) -> ControlAction:
         data["name"]
     )
 
+    check_domain_codomain_spaces(data, ms)
+
     if len(data["codomain"]) == 0:
         data["codomain"] = ("Empty Space",)
 

src/math_spec_mapping/Load/general.py

@@ -159,3 +159,12 @@ def check_json_keys(json: Dict, check_set_key: str) -> None:
 
 def validate_json_schema(json):
     validate(json, schema)
+
+
+def check_domain_codomain_spaces(json: Dict, ms) -> None:
+    if "domain" in json:
+        for key in json["domain"]:
+            assert key in ms["Spaces"], "{} not in spaces".format(key)
+    if "codomain" in json:
+        for key in json["codomain"]:
+            assert key in ms["Spaces"], "{} not in spaces".format(key)

src/math_spec_mapping/Load/mechanism.py

@@ -1,6 +1,6 @@
 from typing import Dict
 from ..Classes import Mechanism
-from .general import check_json_keys
+from .general import check_json_keys, check_domain_codomain_spaces
 
 
 def convert_mechanism(data: Dict, ms: Dict) -> Mechanism:
@@ -26,6 +26,8 @@ def convert_mechanism(data: Dict, ms: Dict) -> Mechanism:
     if len(data["domain"]) == 0:
         data["domain"] = ("Empty Space",)
 
+    check_domain_codomain_spaces(data, ms)
+
     # Copy
     data = data.copy()
 

src/math_spec_mapping/Load/policy.py

@@ -2,7 +2,7 @@
 from ..Classes import Policy, PolicyOption
 
 
-from .general import check_json_keys
+from .general import check_json_keys, check_domain_codomain_spaces
 
 
 def convert_policy_options(data: Dict, ms) -> PolicyOption:
@@ -58,6 +58,7 @@ def convert_policy(data: Dict, ms: Dict) -> Policy:
     assert type(data["domain"]) == tuple, "{} domain is not a tuple".format(
         data["name"]
     )
+    check_domain_codomain_spaces(data, ms)
 
     if len(data["codomain"]) == 0:
         data["codomain"] = ("Empty Space",)

src/math_spec_mapping/Load/stateful_metrics.py

@@ -40,6 +40,12 @@ def convert_stateful_metric(ms, data: Dict) -> StatefulMetricSet:
                 x[1]
             )
 
+        assert (
+            var["type"] in ms["Types"]
+        ), "{} type referenced by {} is not present in math spec".format(
+            var["type"], var["name"]
+        )
+
         var["implementations"] = {}
         if "python" in ms["Implementations"]:
             if "stateful_metrics" in ms["Implementations"]["python"]:

src/math_spec_mapping/Reports/html.py

@@ -169,6 +169,13 @@ def write_spec_tree(
             else:
                 out += symbol3 + "{}".format(var.name) + "\n"
 
+    out += symbol1 + "**Metrics**\n"
+    for name in ms.metrics.keys():
+        if linking:
+            out += symbol2 + "[[{}]]".format(name) + "\n"
+        else:
+            out += symbol2 + "{}".format(name) + "\n"
+
     out += symbol1 + "**Types**\n"
     for name in ms.types.keys():
         if linking:
@@ -221,6 +228,12 @@ def write_spec_tree(
             out += symbol2 + "[[{}]]".format(name) + "\n"
         else:
             out += symbol2 + name + "\n"
+    out += symbol1 + "**Wirings**\n"
+    for name in ms.wiring.keys():
+        if linking:
+            out += symbol2 + "[[{}]]".format(name) + "\n"
+        else:
+            out += symbol2 + name + "\n"
 
     if add_tabbing:
         out = out.split("\n")

src/math_spec_mapping/Reports/markdown.py

@@ -67,7 +67,10 @@ def write_entity_markdown_report(ms, path, entity, add_metadata=True):
 
 
 def write_source_code_block(component, text, path):
-    file_path = component.source_code_location
+    if hasattr(component, "source_code_location"):
+        file_path = component.source_code_location
+    else:
+        file_path = component["Source Code Location"]
     if file_path:
         file_path = os.path.relpath(file_path, path)
         text += "## Spec Source Code Location\n\n"
@@ -939,7 +942,10 @@ def write_wiring_display_markdown_report(ms, path, wiring, add_metadata=True):
         out += "\n"
     out += "\n"
 
-    with open("{}/Displays/Wiring/{}.md".format(path, wiring["name"]), "w") as f:
+    path = "{}/Displays/Wiring/{}.md".format(path, wiring["name"])
+    out = write_source_code_block(wiring, out, path)
+
+    with open(path, "w") as f:
         f.write(out)