From 86b4dda18f1920418b0cd5d124fc030df2e2a6bc Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Fri, 20 Sep 2024 13:53:15 -0400 Subject: [PATCH 01/46] some experimenting --- autotest/test_2298.py | 493 ++++++++++++++++++++ flopy/mf6/templates/package/attrs.jinja | 12 + flopy/mf6/templates/package/docstring.jinja | 112 +++++ flopy/mf6/templates/package/init.jinja | 20 + flopy/mf6/templates/package/package.jinja | 16 + 5 files changed, 653 insertions(+) create mode 100644 autotest/test_2298.py create mode 100644 flopy/mf6/templates/package/attrs.jinja create mode 100644 flopy/mf6/templates/package/docstring.jinja create mode 100644 flopy/mf6/templates/package/init.jinja create mode 100644 flopy/mf6/templates/package/package.jinja diff --git a/autotest/test_2298.py b/autotest/test_2298.py new file mode 100644 index 0000000000..c6244df670 --- /dev/null +++ b/autotest/test_2298.py @@ -0,0 +1,493 @@ +from os import PathLike +from types import NoneType +from typing import ( + Any, + Dict, + ForwardRef, + List, + Literal, + Optional, + Tuple, + Union, + get_args, + get_origin, +) + +import numpy as np +import pytest +from modflow_devtools.misc import run_cmd +from numpy.typing import NDArray + +from autotest.conftest import get_project_root_path + +PROJ_ROOT = get_project_root_path() +DFNS_PATH = PROJ_ROOT / "flopy" / "mf6" / "data" / "dfn" +SCALAR_TYPES = { + "keyword": bool, + "integer": int, + "double precision": float, + "string": str, +} +NP_SCALAR_TYPES = { + "keyword": np.bool_, + "integer": np.int_, + "double precision": np.float64, + "string": np.str_, +} + +Array = NDArray +Scalar = Union[bool, int, float, str] +Definition = Dict[str, Dict[str, Scalar]] + + +def fullname(t: type) -> str: + """Convert a type to a fully qualified name suitable for templating.""" + origin = get_origin(t) + args = get_args(t) + if origin is Union: + if len(args) == 2 and args[1] is NoneType: + return f"typing.{Optional.__name__}[{fullname(args[0])}]" + return f"typing.{Union.__name__}[{', '.join([fullname(a) for a in args])}]" + if origin is tuple: + return f"typing.{Tuple.__name__}[{', '.join([fullname(a) for a in args])}]" + elif origin is list: + return ( + f"typing.{List.__name__}[{', '.join([fullname(a) for a in args])}]" + ) + elif origin is np.ndarray: + return f"NDArray[np.{fullname(args[1].__args__[0])}]" + elif origin is np.dtype: + return str(t) + elif isinstance(t, ForwardRef): + return t.__forward_arg__ + elif t is Ellipsis: + return "..." + elif isinstance(t, type): + return t.__qualname__ + else: + return str(t) + + +def load_dfn(f) -> Tuple[Definition, List[str]]: + """ + Load an input definition file. Returns a tuple containing + a dictionary variables and a list of metadata attributes. + """ + meta = list() + vars = dict() + var = dict() + + for line in f: + # remove whitespace/etc from the line + line = line.strip() + + # record flopy metadata attributes but + # skip all other comment lines + if line.startswith("#"): + _, sep, tail = line.partition("flopy") + if sep == "flopy": + meta.append(tail.strip()) + continue + + # if we hit a newline and the parameter dict + # is nonempty, we've reached the end of its + # block of attributes + if not any(line): + if any(var): + vars[var["name"]] = var + var = dict() + continue + + # split the attribute's key and value and + # store it in the parameter dictionary + key, _, value = line.partition(" ") + var[key] = value + + # add the final parameter + if any(var): + vars[var["name"]] = var + + return vars, meta + + +def test_load_dfn_gwf_ic(): + dfn_path = DFNS_PATH / "gwf-ic.dfn" + with open(dfn_path, "r") as f: + vars, meta = load_dfn(f) + + assert len(vars) == 2 + assert set(vars.keys()) == {"export_array_ascii", "strt"} + assert not any(meta) + + +def test_load_dfn_prt_prp(): + dfn_path = DFNS_PATH / "prt-prp.dfn" + with open(dfn_path, "r") as f: + vars, meta = load_dfn(f) + + assert len(vars) == 40 + assert len(meta) == 1 + + +def get_template_context( + component: str, + subcomponent: str, + definition: Definition, + metadata: List[str], +) -> dict: + """ + Convert an input definition to a template rendering context. + + TODO: pull out a class for the input definition, and expose + this as an instance method? + """ + + def _map_var(var: dict, wrap: bool = False) -> dict: + """ + Transform a variable from its original representation in + an input definition to a form suitable for type hints and + and docstrings. + + This involves expanding nested type hierarchies, converting + input types to equivalent Python primitives and composites, + and various other shaping. + + Notes + ----- + If a `default_value` is not provided, keywords are `False` + by default. Everything else is `None` by default. + + If `wrap` is true, scalars will be wrapped as records with + keywords represented as string literals. This is useful for + unions, to distinguish between choices having the same type. + """ + var_ = { + **var, + # some flags the template uses for formatting. + # these are ofc derivable in Python but Jinja + # doesn't allow arbitrary expressions, and it + # doesn't seem to have `subclass`-ish filters. + # (we convert the variable type to string too + # before returning, for the same reason.) + "is_array": False, + "is_list": False, + "is_record": False, + "is_union": False, + "is_variadic": False, + } + type_ = var["type"] + shape = var.get("shape", None) + shape = None if shape == "" else shape + + # utilities for generating records + # as named tuples. + + def _get_record_fields(name: str) -> dict: + """ + Call `_map_var` recursively on each field + of the record variable with the given name. + + Notes + ----- + This function is provided because records + need extra processing; we remove keywords + and 'filein'/'fileout', which are details + of the mf6io format, not of python/flopy. + """ + record = definition[name] + names = record["type"].split()[1:] + fields = { + n: {**_map_var(field), "optional": field.get("optional", True)} + for n, field in definition.items() + if n in names + } + field_names = list(fields.keys()) + + # if the record represents a file... + if "file" in name: + # remove filein/fileout + for term in ["filein", "fileout"]: + if term in field_names: + fields.pop(term) + + # remove leading keyword + keyword = next(iter(fields), None) + if keyword: + fields.pop(keyword) + + # should just have one remaining field, the file path + n, path = fields.popitem() + if any(fields): + raise ValueError( + f"File record has too many fields: {fields}" + ) + path["type"] = PathLike + fields[n] = path + + # if tagged, remove the leading keyword + elif record.get("tagged", False): + keyword = next(iter(fields), None) + if keyword: + fields.pop(keyword) + + return fields + + # list input can have records or unions as rows. + # lists which have a consistent record type are + # regular, inconsistent record types irregular. + if type_.startswith("recarray"): + # make sure columns are defined + names = type_.split()[1:] + n_names = len(names) + if n_names < 1: + raise ValueError(f"Missing recarray definition: {type_}") + + # regular tabular/columnar data (1 record type) can be + # defined with a nested record (i.e. explicit) or with + # fields directly inside the recarray (implicit). list + # data for unions/keystrings necessarily comes nested. + + def _is_explicit_record(): + return len(names) == 1 and definition[names[0]][ + "type" + ].startswith("record") + + def _is_implicit_record(): + types = [ + fullname(v["type"]) + for n, v in definition.items() + if n in names + ] + scalar_types = list(SCALAR_TYPES.keys()) + return all(t in scalar_types for t in types) + + if _is_explicit_record(): + name = names[0] + record_type = _map_var(definition[name]) + var_["type"] = List[record_type["type"]] + var_["children"] = {name: record_type} + var_["is_list"] = True + elif _is_implicit_record(): + # record implicitly defined, make it on the fly + name = var["name"] + fields = _get_record_fields(name) + record_type = Tuple[ + tuple([f["type"] for f in fields.values()]) + ] + record = { + "name": name, + "type": record_type, + "children": fields, + "is_array": False, + "is_record": True, + "is_union": False, + "is_list": False, + "is_variadic": False, + } + var_["type"] = List[record_type] + var_["children"] = {name: record} + var_["is_list"] = True + else: + # irregular recarray, rows can be any of several types + children = {n: _map_var(definition[n]) for n in names} + var_["type"] = List[ + Union[tuple([c["type"] for c in children.values()])] + ] + var_["children"] = children + var_["is_list"] = True + + # now the basic composite types... + # union (product) type, children are choices of records + elif type_.startswith("keystring"): + names = type_.split()[1:] + children = {n: _map_var(definition[n], wrap=True) for n in names} + var_["type"] = Union[tuple([c["type"] for c in children.values()])] + var_["children"] = children + var_["is_union"] = True + + # record (sum) type, children are fields + elif type_.startswith("record"): + name = var["name"] + fields = _get_record_fields(name) + if len(fields) > 1: + record_type = Tuple[ + tuple([c["type"] for c in fields.values()]) + ] + elif len(fields) == 1: + t = list(fields.values())[0]["type"] + # make sure we don't double-wrap tuples + record_type = t if t is tuple else Tuple[(t,)] + # TODO: if record has 1 field, accept value directly? + var_["type"] = record_type + var_["children"] = fields + var_["is_record"] = True + + # are we wrapping a choice in a union? + # if so, make it a literal if just one + # single keyword, otherwise, repeating + # tuple of strings + elif wrap: + name = var["name"] + field = _map_var(var) + fields = {name: field} + # TODO: there is no way to represent a variadic tuple + # of different leading types (i.e., with only the last + # repeating).. could do: + # - `Tuple[Union[Literal, T], ...]`? + # wrapped_type = Tuple[Union[Literal[name], field["type"]], ...] + # - `Tuple[Literal, Tuple[T, ...]]`? + # ... + # field_type = Literal[name] if field["type"] is bool else wrapped_typed + field_type = ( + Tuple[Literal[name]] + if field["type"] is bool + else Tuple[Any, ...] + ) + fields[name] = {**field, "type": field_type} + var_["type"] = field_type + var_["children"] = fields + var_["is_record"] = True + + # at this point, if it has a shape, it's an array. + # but if it's in a record use a tuple. + elif shape is not None: + if var.get("in_record", False): + if type_ not in SCALAR_TYPES.keys(): + raise TypeError(f"Unsupported repeating type: {type_}") + var_["type"] = Tuple[SCALAR_TYPES[type_], ...] + var_["is_variadic"] = True + elif type_ == "string": + var_["type"] = Tuple[SCALAR_TYPES[type_], ...] + var_["is_variadic"] = True + else: + if type_ not in NP_SCALAR_TYPES.keys(): + raise TypeError(f"Unsupported array type: {type_}") + var_["type"] = NDArray[NP_SCALAR_TYPES[type_]] + var_["is_array"] = True + + # finally a bog standard scalar + else: + var_["type"] = SCALAR_TYPES[type_] + + # wrap with optional if needed + if var_.get("optional", True): + var_["type"] = ( + Optional[var_["type"]] + if ( + var_["type"] is not bool + and var_.get("optional", True) + and not var_.get("in_record", False) + and not wrap + ) + else var_["type"] + ) + + # keywords default to False, everything else to None + var_["default"] = var.pop( + "default", False if var_["type"] is bool else None + ) + + return var_ + + def _qualify(var: dict) -> dict: + """ + Recursively convert the variable's type to a fully qualified string. + """ + + var["type"] = fullname(var["type"]) + children = var.get("children", dict()) + if any(children): + var["children"] = {n: _qualify(c) for n, c in children.items()} + return var + + def _variables(vars: dict) -> dict: + return { + name: _qualify(_map_var(var)) + for name, var in vars.items() + # filter components of composites + # since we've inflated the parent + # types in the hierarchy already + if not var.get("in_record", False) + } + + def _dfn(vars: dict, meta: list) -> list: + """ + Currently, generated classes have a `.dfn` property that + reproduces the corresponding DFN sans a few attributes. + """ + + def _var_dfn(var: dict) -> List[str]: + exclude = ["longname", "description"] + return [ + " ".join([k, v]) for k, v in var.items() if k not in exclude + ] + + return [["header"] + [attr for attr in meta]] + [ + _var_dfn(var) for var in vars.values() + ] + + return { + "component": component, + "subcomponent": subcomponent, + "variables": _variables(definition), + "dfn": _dfn(definition, metadata), + } + + +@pytest.mark.parametrize( + "dfn, n_flat, n_nested", [("gwf-ic", 2, 2), ("prt-prp", 40, 18)] +) +def test_get_template_context(dfn, n_flat, n_nested): + component, subcomponent = dfn.split("-") + + with open(DFNS_PATH / f"{dfn}.dfn") as f: + variables, metadata = load_dfn(f) + + context = get_template_context( + component, subcomponent, variables, metadata + ) + assert context["component"] == component + assert context["subcomponent"] == subcomponent + assert len(context["variables"]) == n_nested + assert len(context["dfn"]) == n_flat + 1 # +1 for metadata + + +from jinja2 import Environment, PackageLoader + +TEMPLATE_ENVS = { + "package": Environment( + loader=PackageLoader("flopy", "mf6/templates/package") + ), +} + + +@pytest.mark.parametrize( + "dfn", + [ + "gwf-ic", + "prt-prp", + "gwe-ctp", + "gwe-cnd", + "gwf-dis", + "prt-mip", + "prt-oc", + ], +) +def test_render_package_template(dfn, function_tmpdir): + component, subcomponent = dfn.split("-") + comp_name = f"{component}{subcomponent}" + comp_type = "package" + environment = TEMPLATE_ENVS[comp_type] + template = environment.get_template(f"{comp_type}.jinja") + + with open(DFNS_PATH / f"{dfn}.dfn", "r") as f: + variables, metadata = load_dfn(f) + + context = get_template_context( + component, subcomponent, variables, metadata + ) + source = template.render(**context) + source_path = function_tmpdir / f"{comp_name}.py" + with open(source_path, "w") as f: + f.write(source) + run_cmd("ruff", "format", source_path, verbose=True) diff --git a/flopy/mf6/templates/package/attrs.jinja b/flopy/mf6/templates/package/attrs.jinja new file mode 100644 index 0000000000..3573443b5d --- /dev/null +++ b/flopy/mf6/templates/package/attrs.jinja @@ -0,0 +1,12 @@ +{# Class attributes for a component class. #} + {% for name, var in variables.items() if var.type %} + {%- if var.is_list or var.is_record %} + {{ var.name }} = ListTemplateGenerator(("{{ component }}6", "{{ subcomponent }}", "{{ var.block }}", "{{ var.name }}")) + {%- elif var.is_array %} + {{ var.name }} = ArrayTemplateGenerator(("{{ component }}6", "{{ subcomponent }}", "{{ var.block }}", "{{ var.name }}")) + {%- endif -%} + {%- endfor %} + package_abbr = "{{ component }}{{ subcomponent }}" + _package_type = "{{ subcomponent }}" + dfn_file_name = "{{ component }}-{{ subcomponent }}.dfn" + dfn = {{ dfn|pprint|indent(10) }} \ No newline at end of file diff --git a/flopy/mf6/templates/package/docstring.jinja b/flopy/mf6/templates/package/docstring.jinja new file mode 100644 index 0000000000..1b18b74aae --- /dev/null +++ b/flopy/mf6/templates/package/docstring.jinja @@ -0,0 +1,112 @@ +{# A component class' docstring. #} + Modflow{{ component.title() }}{{ subcomponent }} defines a {{ subcomponent }} package within a {{ component }}6 model. + + Parameters + ---------- + model : MFModel + Model that this package is a part of. Package is automatically + added to model when it is initialized. + + loading_package : bool + Do not set this variable. It is intended for debugging and internal + processing purposes only. + {% for var in variables.values() %} + {{ var.name }} : {{ var.type }} + {%- if var.description is defined %} + {{ var.description|wordwrap|indent(8) }} + {%- endif -%} + {%- if var.is_record -%} {# record #} + {%- for field in var.children.values() %} + * {{ field.name}} : {{ field.type }} + {%- if field.description is defined %} + {{ field.description|wordwrap|indent(12) }} + {% endif -%} + {%- if field.is_record %} + {%- for f in child.children.values() -%} + * {{ f.name}} : {{ f.type }} + {%- if f.description is defined %} + {{ f.description|wordwrap|indent(12) }} + {%- endif %} + {% endfor -%} + {% endif -%} + {%- if field.is_union %} + {%- for choice in field.children.values() -%} + * {{ choice.name}} : {{ choice.type }} + {%- if choice.is_record %} + {%- if choice.children|length == 1 %} + {{ (choice.children|dictsort()|first)[1].description|wordwrap|indent(16) }} + {%- else -%} + {%- for field in choice.children.values() %} + * {{ field.name}} : {{ field.type }} + {%- if field.description is defined %} + {{ field.description|wordwrap|indent(16) }} + {%- endif %} + {% endfor -%} + {% endif -%} + {%- else %} + {%- if choice.description is defined %} + {{ choice.description|wordwrap|indent(12) }} + {%- endif %} + {%- endif %} + {% endfor -%} + {% endif -%} + {% endfor -%} + {%- elif var.is_union -%} {# union #} + {%- for choice in var.children.values() %} + * {{ choice.name}} : {{ choice.type }} + {%- if choice.is_record %} + {%- for field in choice.children.values() %} + * {{ field.name}} : {{ field.type }} + {%- if field.description is defined %} + {{ field.description|wordwrap|indent(16) }} + {%- endif %} + {%- endfor %} + {%- else %} + {%- if choice.description is defined %} + {{ choice.description|wordwrap|indent(12) }} + {%- endif %} + {%- endif %} + {%- endfor %} + {%- elif var.is_list -%} {# list... #} + {%- for child in var.children.values() %} + {%- if child.is_record -%} {# ...of records #} + {%- for field in child.children.values() -%} + * {{ field.name}} : {{ field.type }} + {%- if field.description is defined %} + {{ field.description|wordwrap|indent(12) }} + {%- endif %} + {% endfor -%} + {%- elif child.is_union -%} {# ...of unions #} + {%- for choice in child.children.values() -%} + * {{ choice.name}} : {{ choice.type }} + {%- if choice.is_record %} + {%- if choice.children|length == 1 %} + {{ (choice.children|dictsort()|first)[1].description|wordwrap|indent(12) }} + {%- else -%} + {%- for field in choice.children.values() %} + * {{ field.name}} : {{ field.type }} + {%- if field.description is defined %} + {{ field.description|wordwrap|indent(16) }} + {%- endif %} + {% endfor -%} + {% endif -%} + {%- else %} + {%- if choice.description is defined %} + {{ choice.description|wordwrap|indent(12) }} + {%- endif %} + {%- endif %} + {% endfor -%} + {%- endif -%} + {% endfor -%} + {%- endif %} + {% endfor %} + filename : str + File name for this package. + + pname : str + Package name for this package. + + parent_file : PathLike + Parent package file that references this package. Only needed for + utility packages (mfutl*). For example, mfutllaktab package must have + a mfgwflak package parent_file. \ No newline at end of file diff --git a/flopy/mf6/templates/package/init.jinja b/flopy/mf6/templates/package/init.jinja new file mode 100644 index 0000000000..d2a97a4b22 --- /dev/null +++ b/flopy/mf6/templates/package/init.jinja @@ -0,0 +1,20 @@ +def __init__( + self, + model: MFModel, + loading_package: bool = False, + {%- for name, var in variables.items() %} + {{ name }}: {{ var.type }} = {{ var.default }}, + {%- endfor %} + filename: typing.Optional[PathLike] = None, + pname: typing.Optional[str] = None, + **kwargs, + ): + super().__init__( + model, "{{ subcomponent }}", filename, pname, loading_package, **kwargs + ) + + # set up variables + {%- for name, var in variables.items() %} + self.{{ name }} = self.build_mfdata("{{ name }}", {{ name }}) + {%- endfor %} + self._init_complete = True \ No newline at end of file diff --git a/flopy/mf6/templates/package/package.jinja b/flopy/mf6/templates/package/package.jinja new file mode 100644 index 0000000000..60e2613397 --- /dev/null +++ b/flopy/mf6/templates/package/package.jinja @@ -0,0 +1,16 @@ +# autogenerated file, do not modify +from os import PathLike +import typing +import numpy as np +from numpy.typing import NDArray +from .. import mfpackage +from ..data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenerator +from flopy.mf6.mfmodel import MFModel + + +class Modflow{{ component.title() }}{{ subcomponent }}(mfpackage.MFPackage): + """{% include "docstring.jinja" %}""" + + {% include "attrs.jinja" %} + + {% include "init.jinja" %} From 2cefd73b64ed298fbc21dd09a946268428524fd0 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 25 Sep 2024 15:17:28 -0400 Subject: [PATCH 02/46] cleaner impl, test more components, proper diagrams in mf6_dev_guide.md --- autotest/test_2298.py | 98 +++++++++------ docs/mf6_dev_guide.md | 52 ++++---- flopy/mf6/templates/{package => }/attrs.jinja | 0 flopy/mf6/templates/docstring.jinja | 23 ++++ flopy/mf6/templates/{package => }/init.jinja | 4 +- .../mf6/templates/{package => }/package.jinja | 1 + flopy/mf6/templates/package/docstring.jinja | 112 ------------------ flopy/mf6/templates/params.jinja | 9 ++ flopy/mf6/templates/simulation.jinja | 99 ++++++++++++++++ 9 files changed, 227 insertions(+), 171 deletions(-) rename flopy/mf6/templates/{package => }/attrs.jinja (100%) create mode 100644 flopy/mf6/templates/docstring.jinja rename flopy/mf6/templates/{package => }/init.jinja (85%) rename flopy/mf6/templates/{package => }/package.jinja (86%) delete mode 100644 flopy/mf6/templates/package/docstring.jinja create mode 100644 flopy/mf6/templates/params.jinja create mode 100644 flopy/mf6/templates/simulation.jinja diff --git a/autotest/test_2298.py b/autotest/test_2298.py index c6244df670..57e0f951b4 100644 --- a/autotest/test_2298.py +++ b/autotest/test_2298.py @@ -1,5 +1,6 @@ +from enum import Enum +from keyword import kwlist from os import PathLike -from types import NoneType from typing import ( Any, Dict, @@ -41,19 +42,21 @@ def fullname(t: type) -> str: - """Convert a type to a fully qualified name suitable for templating.""" + """Convert a type to a name suitable for templating.""" origin = get_origin(t) args = get_args(t) - if origin is Union: - if len(args) == 2 and args[1] is NoneType: - return f"typing.{Optional.__name__}[{fullname(args[0])}]" - return f"typing.{Union.__name__}[{', '.join([fullname(a) for a in args])}]" - if origin is tuple: - return f"typing.{Tuple.__name__}[{', '.join([fullname(a) for a in args])}]" - elif origin is list: + if origin is Literal: return ( - f"typing.{List.__name__}[{', '.join([fullname(a) for a in args])}]" + f"{Literal.__name__}[{', '.join(['"' + a + '"' for a in args])}]" ) + elif origin is Union: + if len(args) == 2 and args[1] is type(None): + return f"{Optional.__name__}[{fullname(args[0])}]" + return f"{Union.__name__}[{', '.join([fullname(a) for a in args])}]" + elif origin is tuple: + return f"{Tuple.__name__}[{', '.join([fullname(a) for a in args])}]" + elif origin is list: + return f"{List.__name__}[{', '.join([fullname(a) for a in args])}]" elif origin is np.ndarray: return f"NDArray[np.{fullname(args[1].__args__[0])}]" elif origin is np.dtype: @@ -174,7 +177,9 @@ def _map_var(var: dict, wrap: bool = False) -> dict: "is_record": False, "is_union": False, "is_variadic": False, + "is_choice": False, } + name_ = var["name"] type_ = var["type"] shape = var.get("shape", None) shape = None if shape == "" else shape @@ -269,7 +274,7 @@ def _is_implicit_record(): var_["is_list"] = True elif _is_implicit_record(): # record implicitly defined, make it on the fly - name = var["name"] + name = name_ fields = _get_record_fields(name) record_type = Tuple[ tuple([f["type"] for f in fields.values()]) @@ -283,6 +288,7 @@ def _is_implicit_record(): "is_union": False, "is_list": False, "is_variadic": False, + "is_choice": False, } var_["type"] = List[record_type] var_["children"] = {name: record} @@ -307,7 +313,7 @@ def _is_implicit_record(): # record (sum) type, children are fields elif type_.startswith("record"): - name = var["name"] + name = name_ fields = _get_record_fields(name) if len(fields) > 1: record_type = Tuple[ @@ -316,7 +322,7 @@ def _is_implicit_record(): elif len(fields) == 1: t = list(fields.values())[0]["type"] # make sure we don't double-wrap tuples - record_type = t if t is tuple else Tuple[(t,)] + record_type = t if get_origin(t) is tuple else Tuple[(t,)] # TODO: if record has 1 field, accept value directly? var_["type"] = record_type var_["children"] = fields @@ -327,26 +333,22 @@ def _is_implicit_record(): # single keyword, otherwise, repeating # tuple of strings elif wrap: - name = var["name"] + name = name_ field = _map_var(var) fields = {name: field} - # TODO: there is no way to represent a variadic tuple - # of different leading types (i.e., with only the last - # repeating).. could do: - # - `Tuple[Union[Literal, T], ...]`? - # wrapped_type = Tuple[Union[Literal[name], field["type"]], ...] - # - `Tuple[Literal, Tuple[T, ...]]`? - # ... - # field_type = Literal[name] if field["type"] is bool else wrapped_typed field_type = ( + Literal[name] if field["type"] is bool else field["type"] + ) + record_type = ( Tuple[Literal[name]] if field["type"] is bool - else Tuple[Any, ...] + else Tuple[Literal[name], field["type"]] ) fields[name] = {**field, "type": field_type} - var_["type"] = field_type + var_["type"] = record_type var_["children"] = fields var_["is_record"] = True + var_["is_choice"] = True # at this point, if it has a shape, it's an array. # but if it's in a record use a tuple. @@ -369,7 +371,7 @@ def _is_implicit_record(): else: var_["type"] = SCALAR_TYPES[type_] - # wrap with optional if needed + # make optional if needed if var_.get("optional", True): var_["type"] = ( Optional[var_["type"]] @@ -387,6 +389,15 @@ def _is_implicit_record(): "default", False if var_["type"] is bool else None ) + # remove backslashes from description + var_["description"] = var_.get("description", "").replace("\\", "") + + # if name is a reserved keyword, + # add trailing underscore to it + var_["name"] = ( + f"{var_['name']}_" if var_["name"] in kwlist else var_["name"] + ) + return var_ def _qualify(var: dict) -> dict: @@ -454,16 +465,26 @@ def test_get_template_context(dfn, n_flat, n_nested): from jinja2 import Environment, PackageLoader -TEMPLATE_ENVS = { - "package": Environment( - loader=PackageLoader("flopy", "mf6/templates/package") - ), -} + +class ComponentType(Enum): + Package = "package" + Simulation = "simulation" + + +TEMPLATE_ENV = Environment(loader=PackageLoader("flopy", "mf6/templates/")) + + +def get_component_type(component, subcomponent) -> ComponentType: + if component == "sim" and subcomponent == "nam": + return ComponentType.Simulation + else: + return ComponentType.Package @pytest.mark.parametrize( "dfn", [ + # packages "gwf-ic", "prt-prp", "gwe-ctp", @@ -471,14 +492,23 @@ def test_get_template_context(dfn, n_flat, n_nested): "gwf-dis", "prt-mip", "prt-oc", + # models + "gwf-nam", + "gwt-nam", + "gwe-nam", + "prt-nam", + # solutions + "sln-ims", + "sln-ems", + # simulation + "sim-nam", ], ) -def test_render_package_template(dfn, function_tmpdir): +def test_render_template(dfn, function_tmpdir): component, subcomponent = dfn.split("-") comp_name = f"{component}{subcomponent}" - comp_type = "package" - environment = TEMPLATE_ENVS[comp_type] - template = environment.get_template(f"{comp_type}.jinja") + comp_type = get_component_type(component, subcomponent).value + template = TEMPLATE_ENV.get_template(f"{comp_type}.jinja") with open(DFNS_PATH / f"{dfn}.dfn", "r") as f: variables, metadata = load_dfn(f) diff --git a/docs/mf6_dev_guide.md b/docs/mf6_dev_guide.md index 61c364d3de..8f95ee9bbb 100644 --- a/docs/mf6_dev_guide.md +++ b/docs/mf6_dev_guide.md @@ -10,12 +10,17 @@ FPMF6 uses meta-data files located in flopy/mf6/data/dfn to define the model and All meta-data can be accessed from the flopy.mf6.data.mfstructure.MFStructure class. This is a singleton class, meaning only one instance of this class can be created. The class contains a sim_struct attribute (which is a flopy.mf6.data.mfstructure.MFSimulationStructure object) which contains all of the meta-data for all package files. Meta-data is stored in a structured format. MFSimulationStructure contains MFModelStructure and MFInputFileStructure objects, which contain the meta-data for each model type and each "simulation-level" package (tdis, ims, ...). MFModelStructure contains model specific meta-data and a MFInputFileStructure object for each package in that model. MFInputFileStructure contains package specific meta-data and a MFBlockStructure object for each block contained in the package file. MFBlockStructure contains block specific meta-data and a MFDataStructure object for each data structure defined in the block, and MFDataStructure contains data structure specific meta-data and a MFDataItemStructure object for each data item contained in the data structure. Data structures define the structure of data that is naturally grouped together, for example, the data in a numpy recarray. Data item structures define the structure of specific pieces of data, for example, a single column of a numpy recarray. The meta-data defined in these classes provides all the information FloPy needs to read and write MODFLOW 6 package and name files, create the Flopy interface, and check the data for various constraints. - -*** -MFStructure --+ MFSimulationStructure --+ MFModelStructure --+ MFInputFileStructure --+ MFBlockStructure --+ MFDataStructure --+ MFDataItemStructure - -Figure 1: FPMF6 generic data structure classes. Lines connecting classes show a relationship defined between the two connected classes. A "*" next to the class means that the class is a sub-class of the connected class. A "+" next to the class means that the class is contained within the connected class. -*** +```mermaid +classDiagram + MFStructure --* "1" MFSimulationStructure : has + MFSimulationStructure --* "1+" MFModelStructure : has + MFModelStructure --* "1" MFInputFileStructure : has + MFInputFileStructure --* "1+" MFBlockStructure : has + MFBlockStructure --* "1+" MFDataStructure : has + MFDataStructure --* "1+" MFDataItemStructure : has +``` + +Figure 1: Generic data structure hierarchy. Connections show composition relationships. Package and Data Base Classes ----------------------------------------------- @@ -23,25 +28,26 @@ Package and Data Base Classes The package and data classes are related as shown below in figure 2. On the top of the figure 2 is the MFPackage class, which is the base class for all packages. MFPackage contains generic methods for building data objects and reading and writing the package to a file. MFPackage contains a MFInputFileStructure object that defines how the data is structured in the package file. MFPackage also contains a dictionary of blocks (MFBlock). The MFBlock class is a generic class used to represent a block within a package. MFBlock contains a MFBlockStructure object that defines how the data in the block is structured. MFBlock also contains a dictionary of data objects (subclasses of MFData) contained in the block and a list of block headers (MFBlockHeader) for that block. Block headers contain the block's name and optionally data items (eg. iprn). -*** -MFPackage --+ MFBlock --+ MFData - -MFPackage --+ MFInputFileStructure - -MFBlock --+ MFBlockStructure - -MFData --+ MFDataStructure - -MFData --* MFArray --* MFTransientArray - -MFData --* MFList --* MFTransientList - -MFData --* MFScalar --* MFTransientScalar - -MFTransientData --* MFTransientArray, MFTransientList, MFTransientScalar +```mermaid +classDiagram + +MFPackage --* "1+" MFBlock : has +MFBlock --* "1+" MFData : has +MFPackage --* "1" MFInputFileStructure : has +MFBlock --* "1" MFBlockStructure : has +MFData --* "1" MFDataStructure : has +MFData --|> MFArray +MFArray --|> MFTransientArray +MFData --|> MFList +MFList --|> MFTransientList +MFData --|> MFScalar +MFScalar --|> MFTransientScalar +MFTransientData --|> MFTransientArray +MFTransientData --|> MFTransientList +MFTransientData --|> MFTransientScalar +``` Figure 2: FPMF6 package and data classes. Lines connecting classes show a relationship defined between the two connected classes. A "*" next to the class means that the class is a sub-class of the connected class. A "+" next to the class means that the class is contained within the connected class. -*** There are three main types of data, MFList, MFArray, and MFScalar data. All three of these data types are derived from the MFData abstract base class. MFList data is the type of data stored in a spreadsheet with different column headings. For example, the data describing a flow barrier are of type MFList. MFList data is stored in numpy recarrays. MFArray data is data of a single type (eg. all integer values). For example, the model's HK values are of type MFArray. MFArrays are stored in numpy ndarrays. MFScalar data is a single data item. Most MFScalar data are options. All MFData subclasses contain an MFDataStructure object that defines the expected structure and types of the data. diff --git a/flopy/mf6/templates/package/attrs.jinja b/flopy/mf6/templates/attrs.jinja similarity index 100% rename from flopy/mf6/templates/package/attrs.jinja rename to flopy/mf6/templates/attrs.jinja diff --git a/flopy/mf6/templates/docstring.jinja b/flopy/mf6/templates/docstring.jinja new file mode 100644 index 0000000000..b437e90096 --- /dev/null +++ b/flopy/mf6/templates/docstring.jinja @@ -0,0 +1,23 @@ +{# A component class' docstring. #} + Modflow{{ component.title() }}{{ subcomponent }} defines a {{ subcomponent }} package within a {{ component }}6 model. + + Parameters + ---------- + model : MFModel + Model that this package is a part of. Package is automatically + added to model when it is initialized. + + loading_package : bool + Do not set this variable. It is intended for debugging and internal + processing purposes only. + {% include "params.jinja" %} + filename : str + File name for this package. + + pname : str + Package name for this package. + + parent_file : PathLike + Parent package file that references this package. Only needed for + utility packages (mfutl*). For example, mfutllaktab package must have + a mfgwflak package parent_file. \ No newline at end of file diff --git a/flopy/mf6/templates/package/init.jinja b/flopy/mf6/templates/init.jinja similarity index 85% rename from flopy/mf6/templates/package/init.jinja rename to flopy/mf6/templates/init.jinja index d2a97a4b22..94f646d066 100644 --- a/flopy/mf6/templates/package/init.jinja +++ b/flopy/mf6/templates/init.jinja @@ -5,8 +5,8 @@ def __init__( {%- for name, var in variables.items() %} {{ name }}: {{ var.type }} = {{ var.default }}, {%- endfor %} - filename: typing.Optional[PathLike] = None, - pname: typing.Optional[str] = None, + filename: Optional[PathLike] = None, + pname: Optional[str] = None, **kwargs, ): super().__init__( diff --git a/flopy/mf6/templates/package/package.jinja b/flopy/mf6/templates/package.jinja similarity index 86% rename from flopy/mf6/templates/package/package.jinja rename to flopy/mf6/templates/package.jinja index 60e2613397..30d22df04f 100644 --- a/flopy/mf6/templates/package/package.jinja +++ b/flopy/mf6/templates/package.jinja @@ -2,6 +2,7 @@ from os import PathLike import typing import numpy as np +from typing import Any, Optional, Tuple, List, Dict, Union, Literal from numpy.typing import NDArray from .. import mfpackage from ..data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenerator diff --git a/flopy/mf6/templates/package/docstring.jinja b/flopy/mf6/templates/package/docstring.jinja deleted file mode 100644 index 1b18b74aae..0000000000 --- a/flopy/mf6/templates/package/docstring.jinja +++ /dev/null @@ -1,112 +0,0 @@ -{# A component class' docstring. #} - Modflow{{ component.title() }}{{ subcomponent }} defines a {{ subcomponent }} package within a {{ component }}6 model. - - Parameters - ---------- - model : MFModel - Model that this package is a part of. Package is automatically - added to model when it is initialized. - - loading_package : bool - Do not set this variable. It is intended for debugging and internal - processing purposes only. - {% for var in variables.values() %} - {{ var.name }} : {{ var.type }} - {%- if var.description is defined %} - {{ var.description|wordwrap|indent(8) }} - {%- endif -%} - {%- if var.is_record -%} {# record #} - {%- for field in var.children.values() %} - * {{ field.name}} : {{ field.type }} - {%- if field.description is defined %} - {{ field.description|wordwrap|indent(12) }} - {% endif -%} - {%- if field.is_record %} - {%- for f in child.children.values() -%} - * {{ f.name}} : {{ f.type }} - {%- if f.description is defined %} - {{ f.description|wordwrap|indent(12) }} - {%- endif %} - {% endfor -%} - {% endif -%} - {%- if field.is_union %} - {%- for choice in field.children.values() -%} - * {{ choice.name}} : {{ choice.type }} - {%- if choice.is_record %} - {%- if choice.children|length == 1 %} - {{ (choice.children|dictsort()|first)[1].description|wordwrap|indent(16) }} - {%- else -%} - {%- for field in choice.children.values() %} - * {{ field.name}} : {{ field.type }} - {%- if field.description is defined %} - {{ field.description|wordwrap|indent(16) }} - {%- endif %} - {% endfor -%} - {% endif -%} - {%- else %} - {%- if choice.description is defined %} - {{ choice.description|wordwrap|indent(12) }} - {%- endif %} - {%- endif %} - {% endfor -%} - {% endif -%} - {% endfor -%} - {%- elif var.is_union -%} {# union #} - {%- for choice in var.children.values() %} - * {{ choice.name}} : {{ choice.type }} - {%- if choice.is_record %} - {%- for field in choice.children.values() %} - * {{ field.name}} : {{ field.type }} - {%- if field.description is defined %} - {{ field.description|wordwrap|indent(16) }} - {%- endif %} - {%- endfor %} - {%- else %} - {%- if choice.description is defined %} - {{ choice.description|wordwrap|indent(12) }} - {%- endif %} - {%- endif %} - {%- endfor %} - {%- elif var.is_list -%} {# list... #} - {%- for child in var.children.values() %} - {%- if child.is_record -%} {# ...of records #} - {%- for field in child.children.values() -%} - * {{ field.name}} : {{ field.type }} - {%- if field.description is defined %} - {{ field.description|wordwrap|indent(12) }} - {%- endif %} - {% endfor -%} - {%- elif child.is_union -%} {# ...of unions #} - {%- for choice in child.children.values() -%} - * {{ choice.name}} : {{ choice.type }} - {%- if choice.is_record %} - {%- if choice.children|length == 1 %} - {{ (choice.children|dictsort()|first)[1].description|wordwrap|indent(12) }} - {%- else -%} - {%- for field in choice.children.values() %} - * {{ field.name}} : {{ field.type }} - {%- if field.description is defined %} - {{ field.description|wordwrap|indent(16) }} - {%- endif %} - {% endfor -%} - {% endif -%} - {%- else %} - {%- if choice.description is defined %} - {{ choice.description|wordwrap|indent(12) }} - {%- endif %} - {%- endif %} - {% endfor -%} - {%- endif -%} - {% endfor -%} - {%- endif %} - {% endfor %} - filename : str - File name for this package. - - pname : str - Package name for this package. - - parent_file : PathLike - Parent package file that references this package. Only needed for - utility packages (mfutl*). For example, mfutllaktab package must have - a mfgwflak package parent_file. \ No newline at end of file diff --git a/flopy/mf6/templates/params.jinja b/flopy/mf6/templates/params.jinja new file mode 100644 index 0000000000..32a20005bd --- /dev/null +++ b/flopy/mf6/templates/params.jinja @@ -0,0 +1,9 @@ +{% for var in variables.values() recursive %} + {% if loop.depth > 1 %}* {% endif %}{{ var.name }} : {{ var.type }} +{%- if var.description is defined and not var.is_choice %} +{{ var.description|wordwrap|indent(4 + (loop.depth * 4), first=True) }} +{%- endif %} +{%- if var.children is defined -%} +{{ loop(var.children.values())|indent(4) }} +{%- endif %} +{% endfor %} \ No newline at end of file diff --git a/flopy/mf6/templates/simulation.jinja b/flopy/mf6/templates/simulation.jinja new file mode 100644 index 0000000000..a85ff9a657 --- /dev/null +++ b/flopy/mf6/templates/simulation.jinja @@ -0,0 +1,99 @@ +# autogenerated file, do not modify +import os +from typing import Union + +from .. import mfsimbase + + +class MFSimulation(mfsimbase.MFSimulationBase): + """ + MFSimulation is used to load, build, and/or save a MODFLOW 6 simulation. + A MFSimulation object must be created before creating any of the MODFLOW 6 + model objects. + + Parameters + ---------- + sim_name : str + Name of the simulation + {% include "params.jinja" %} + + Methods + ------- + load : (sim_name : str, version : string, + exe_name : str or PathLike, sim_ws : str or PathLike, strict : bool, + verbosity_level : int, load_only : list, verify_data : bool, + write_headers : bool, lazy_io : bool, use_pandas : bool, + ) : MFSimulation + a class method that loads a simulation from files + """ + + def __init__( + self, + sim_name="sim", + version="mf6", + exe_name: Union[str, os.PathLike] = "mf6", + sim_ws: Union[str, os.PathLike] = os.curdir, + verbosity_level=1, + write_headers=True, + use_pandas=True, + lazy_io=False, + continue_=None, + nocheck=None, + memory_print_option=None, + maxerrors=None, + print_input=None, + hpc_data=None, + ): + super().__init__( + sim_name=sim_name, + version=version, + exe_name=exe_name, + sim_ws=sim_ws, + verbosity_level=verbosity_level, + write_headers=write_headers, + lazy_io=lazy_io, + use_pandas=use_pandas, + ) + + self.name_file.continue_.set_data(continue_) + self.name_file.nocheck.set_data(nocheck) + self.name_file.memory_print_option.set_data(memory_print_option) + self.name_file.maxerrors.set_data(maxerrors) + self.name_file.print_input.set_data(print_input) + + self.continue_ = self.name_file.continue_ + self.nocheck = self.name_file.nocheck + self.memory_print_option = self.name_file.memory_print_option + self.maxerrors = self.name_file.maxerrors + self.print_input = self.name_file.print_input + self.hpc_data = self._create_package("hpc", hpc_data) + + @classmethod + def load( + cls, + sim_name="modflowsim", + version="mf6", + exe_name: Union[str, os.PathLike] = "mf6", + sim_ws: Union[str, os.PathLike] = os.curdir, + strict=True, + verbosity_level=1, + load_only=None, + verify_data=False, + write_headers=True, + lazy_io=False, + use_pandas=True, + ): + return mfsimbase.MFSimulationBase.load( + cls, + sim_name, + version, + exe_name, + sim_ws, + strict, + verbosity_level, + load_only, + verify_data, + write_headers, + lazy_io, + use_pandas, + ) From 05b1fdba22c3015975ad0de3d8e23d199a1fdbe5 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 25 Sep 2024 15:22:55 -0400 Subject: [PATCH 03/46] cleanup --- autotest/test_2298.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/autotest/test_2298.py b/autotest/test_2298.py index 57e0f951b4..6bbeecc783 100644 --- a/autotest/test_2298.py +++ b/autotest/test_2298.py @@ -329,9 +329,8 @@ def _is_implicit_record(): var_["is_record"] = True # are we wrapping a choice in a union? - # if so, make it a literal if just one - # single keyword, otherwise, repeating - # tuple of strings + # if so, use a literal for the leading + # keyword like tuple (Literal[...], T) elif wrap: name = name_ field = _map_var(var) @@ -351,7 +350,7 @@ def _is_implicit_record(): var_["is_choice"] = True # at this point, if it has a shape, it's an array. - # but if it's in a record use a tuple. + # but if it's in a record use a variadic tuple. elif shape is not None: if var.get("in_record", False): if type_ not in SCALAR_TYPES.keys(): From 4f19763fbd62dd505619a0c552de05ca35d3431c Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 25 Sep 2024 15:24:52 -0400 Subject: [PATCH 04/46] appease python 3.9? --- autotest/test_2298.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/autotest/test_2298.py b/autotest/test_2298.py index 6bbeecc783..322257ee16 100644 --- a/autotest/test_2298.py +++ b/autotest/test_2298.py @@ -46,8 +46,9 @@ def fullname(t: type) -> str: origin = get_origin(t) args = get_args(t) if origin is Literal: + args = ['"' + a + '"' for a in args] return ( - f"{Literal.__name__}[{', '.join(['"' + a + '"' for a in args])}]" + f"{Literal.__name__}[{', '.join(args)}]" ) elif origin is Union: if len(args) == 2 and args[1] is type(None): From 205c1826b980e1430d52e9e07abbca7c43ad700f Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 25 Sep 2024 16:07:54 -0400 Subject: [PATCH 05/46] ruff, kw/record tagging fix --- autotest/test_2298.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/autotest/test_2298.py b/autotest/test_2298.py index 322257ee16..052fb158ac 100644 --- a/autotest/test_2298.py +++ b/autotest/test_2298.py @@ -47,9 +47,7 @@ def fullname(t: type) -> str: args = get_args(t) if origin is Literal: args = ['"' + a + '"' for a in args] - return ( - f"{Literal.__name__}[{', '.join(args)}]" - ) + return f"{Literal.__name__}[{', '.join(args)}]" elif origin is Union: if len(args) == 2 and args[1] is type(None): return f"{Optional.__name__}[{fullname(args[0])}]" @@ -369,7 +367,10 @@ def _is_implicit_record(): # finally a bog standard scalar else: - var_["type"] = SCALAR_TYPES[type_] + # if it's a keyword tag for another + # variable, make it a string literal + tag = type_ == "keyword" and (wrap or var.get("tagged", False)) + var_["type"] = Literal[name_] if tag else SCALAR_TYPES[type_] # make optional if needed if var_.get("optional", True): @@ -492,6 +493,7 @@ def get_component_type(component, subcomponent) -> ComponentType: "gwf-dis", "prt-mip", "prt-oc", + "gwf-oc", # models "gwf-nam", "gwt-nam", From fb68caec4e87db69d2c9bcd89fe62e49332d6022 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 25 Sep 2024 16:39:51 -0400 Subject: [PATCH 06/46] common variable substitutions --- autotest/test_2298.py | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/autotest/test_2298.py b/autotest/test_2298.py index 052fb158ac..8e38f47073 100644 --- a/autotest/test_2298.py +++ b/autotest/test_2298.py @@ -134,6 +134,7 @@ def test_load_dfn_prt_prp(): def get_template_context( component: str, subcomponent: str, + common_vars: Definition, definition: Definition, metadata: List[str], ) -> dict: @@ -390,8 +391,23 @@ def _is_implicit_record(): "default", False if var_["type"] is bool else None ) - # remove backslashes from description - var_["description"] = var_.get("description", "").replace("\\", "") + # make substitutions from common variables + # and remove backslashes from description + def _map_descr(description: str) -> str: + description = description.replace("\\", "") + _, replace, tail = description.strip().partition("REPLACE") + if replace: + key, _, replacements = tail.strip().partition(" ") + replacements = eval(replacements) + val = common_vars.get(key, None).get("description", "") + if val is None: + raise ValueError(f"Common variable not found: {key}") + if any(replacements): + return val.replace("{#1}", replacements["{#1}"]) + return val + return description + + var_["description"] = _map_descr(var_.get("description", "")) # if name is a reserved keyword, # add trailing underscore to it @@ -452,11 +468,14 @@ def _var_dfn(var: dict) -> List[str]: def test_get_template_context(dfn, n_flat, n_nested): component, subcomponent = dfn.split("-") + with open(DFNS_PATH / "common.dfn") as f: + common_vars, _ = load_dfn(f) + with open(DFNS_PATH / f"{dfn}.dfn") as f: variables, metadata = load_dfn(f) context = get_template_context( - component, subcomponent, variables, metadata + component, subcomponent, common_vars, variables, metadata ) assert context["component"] == component assert context["subcomponent"] == subcomponent @@ -466,15 +485,14 @@ def test_get_template_context(dfn, n_flat, n_nested): from jinja2 import Environment, PackageLoader +TEMPLATE_ENV = Environment(loader=PackageLoader("flopy", "mf6/templates/")) + class ComponentType(Enum): Package = "package" Simulation = "simulation" -TEMPLATE_ENV = Environment(loader=PackageLoader("flopy", "mf6/templates/")) - - def get_component_type(component, subcomponent) -> ComponentType: if component == "sim" and subcomponent == "nam": return ComponentType.Simulation @@ -490,6 +508,7 @@ def get_component_type(component, subcomponent) -> ComponentType: "prt-prp", "gwe-ctp", "gwe-cnd", + "gwe-esl", "gwf-dis", "prt-mip", "prt-oc", @@ -512,11 +531,14 @@ def test_render_template(dfn, function_tmpdir): comp_type = get_component_type(component, subcomponent).value template = TEMPLATE_ENV.get_template(f"{comp_type}.jinja") + with open(DFNS_PATH / "common.dfn") as f: + common_vars, _ = load_dfn(f) + with open(DFNS_PATH / f"{dfn}.dfn", "r") as f: variables, metadata = load_dfn(f) context = get_template_context( - component, subcomponent, variables, metadata + component, subcomponent, common_vars, variables, metadata ) source = template.render(**context) source_path = function_tmpdir / f"{comp_name}.py" From b78dd38734d19d3eb57b91bf6fb4730d7d6587ee Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 25 Sep 2024 20:27:58 -0400 Subject: [PATCH 07/46] reorg, prep for e2e testing of createpackages.py --- autotest/test_2298.py | 547 ------------------------------ autotest/test_createpackages.py | 70 ++++ autotest/test_dfn.py | 15 + flopy/mf6/templates/model.jinja | 141 ++++++++ flopy/mf6/utils/codegen.py | 0 flopy/mf6/utils/createpackages.py | 487 ++++++++++++++++++++++++++ flopy/mf6/utils/dfn.py | 46 +++ 7 files changed, 759 insertions(+), 547 deletions(-) delete mode 100644 autotest/test_2298.py create mode 100644 autotest/test_createpackages.py create mode 100644 autotest/test_dfn.py create mode 100644 flopy/mf6/templates/model.jinja create mode 100644 flopy/mf6/utils/codegen.py create mode 100644 flopy/mf6/utils/dfn.py diff --git a/autotest/test_2298.py b/autotest/test_2298.py deleted file mode 100644 index 8e38f47073..0000000000 --- a/autotest/test_2298.py +++ /dev/null @@ -1,547 +0,0 @@ -from enum import Enum -from keyword import kwlist -from os import PathLike -from typing import ( - Any, - Dict, - ForwardRef, - List, - Literal, - Optional, - Tuple, - Union, - get_args, - get_origin, -) - -import numpy as np -import pytest -from modflow_devtools.misc import run_cmd -from numpy.typing import NDArray - -from autotest.conftest import get_project_root_path - -PROJ_ROOT = get_project_root_path() -DFNS_PATH = PROJ_ROOT / "flopy" / "mf6" / "data" / "dfn" -SCALAR_TYPES = { - "keyword": bool, - "integer": int, - "double precision": float, - "string": str, -} -NP_SCALAR_TYPES = { - "keyword": np.bool_, - "integer": np.int_, - "double precision": np.float64, - "string": np.str_, -} - -Array = NDArray -Scalar = Union[bool, int, float, str] -Definition = Dict[str, Dict[str, Scalar]] - - -def fullname(t: type) -> str: - """Convert a type to a name suitable for templating.""" - origin = get_origin(t) - args = get_args(t) - if origin is Literal: - args = ['"' + a + '"' for a in args] - return f"{Literal.__name__}[{', '.join(args)}]" - elif origin is Union: - if len(args) == 2 and args[1] is type(None): - return f"{Optional.__name__}[{fullname(args[0])}]" - return f"{Union.__name__}[{', '.join([fullname(a) for a in args])}]" - elif origin is tuple: - return f"{Tuple.__name__}[{', '.join([fullname(a) for a in args])}]" - elif origin is list: - return f"{List.__name__}[{', '.join([fullname(a) for a in args])}]" - elif origin is np.ndarray: - return f"NDArray[np.{fullname(args[1].__args__[0])}]" - elif origin is np.dtype: - return str(t) - elif isinstance(t, ForwardRef): - return t.__forward_arg__ - elif t is Ellipsis: - return "..." - elif isinstance(t, type): - return t.__qualname__ - else: - return str(t) - - -def load_dfn(f) -> Tuple[Definition, List[str]]: - """ - Load an input definition file. Returns a tuple containing - a dictionary variables and a list of metadata attributes. - """ - meta = list() - vars = dict() - var = dict() - - for line in f: - # remove whitespace/etc from the line - line = line.strip() - - # record flopy metadata attributes but - # skip all other comment lines - if line.startswith("#"): - _, sep, tail = line.partition("flopy") - if sep == "flopy": - meta.append(tail.strip()) - continue - - # if we hit a newline and the parameter dict - # is nonempty, we've reached the end of its - # block of attributes - if not any(line): - if any(var): - vars[var["name"]] = var - var = dict() - continue - - # split the attribute's key and value and - # store it in the parameter dictionary - key, _, value = line.partition(" ") - var[key] = value - - # add the final parameter - if any(var): - vars[var["name"]] = var - - return vars, meta - - -def test_load_dfn_gwf_ic(): - dfn_path = DFNS_PATH / "gwf-ic.dfn" - with open(dfn_path, "r") as f: - vars, meta = load_dfn(f) - - assert len(vars) == 2 - assert set(vars.keys()) == {"export_array_ascii", "strt"} - assert not any(meta) - - -def test_load_dfn_prt_prp(): - dfn_path = DFNS_PATH / "prt-prp.dfn" - with open(dfn_path, "r") as f: - vars, meta = load_dfn(f) - - assert len(vars) == 40 - assert len(meta) == 1 - - -def get_template_context( - component: str, - subcomponent: str, - common_vars: Definition, - definition: Definition, - metadata: List[str], -) -> dict: - """ - Convert an input definition to a template rendering context. - - TODO: pull out a class for the input definition, and expose - this as an instance method? - """ - - def _map_var(var: dict, wrap: bool = False) -> dict: - """ - Transform a variable from its original representation in - an input definition to a form suitable for type hints and - and docstrings. - - This involves expanding nested type hierarchies, converting - input types to equivalent Python primitives and composites, - and various other shaping. - - Notes - ----- - If a `default_value` is not provided, keywords are `False` - by default. Everything else is `None` by default. - - If `wrap` is true, scalars will be wrapped as records with - keywords represented as string literals. This is useful for - unions, to distinguish between choices having the same type. - """ - var_ = { - **var, - # some flags the template uses for formatting. - # these are ofc derivable in Python but Jinja - # doesn't allow arbitrary expressions, and it - # doesn't seem to have `subclass`-ish filters. - # (we convert the variable type to string too - # before returning, for the same reason.) - "is_array": False, - "is_list": False, - "is_record": False, - "is_union": False, - "is_variadic": False, - "is_choice": False, - } - name_ = var["name"] - type_ = var["type"] - shape = var.get("shape", None) - shape = None if shape == "" else shape - - # utilities for generating records - # as named tuples. - - def _get_record_fields(name: str) -> dict: - """ - Call `_map_var` recursively on each field - of the record variable with the given name. - - Notes - ----- - This function is provided because records - need extra processing; we remove keywords - and 'filein'/'fileout', which are details - of the mf6io format, not of python/flopy. - """ - record = definition[name] - names = record["type"].split()[1:] - fields = { - n: {**_map_var(field), "optional": field.get("optional", True)} - for n, field in definition.items() - if n in names - } - field_names = list(fields.keys()) - - # if the record represents a file... - if "file" in name: - # remove filein/fileout - for term in ["filein", "fileout"]: - if term in field_names: - fields.pop(term) - - # remove leading keyword - keyword = next(iter(fields), None) - if keyword: - fields.pop(keyword) - - # should just have one remaining field, the file path - n, path = fields.popitem() - if any(fields): - raise ValueError( - f"File record has too many fields: {fields}" - ) - path["type"] = PathLike - fields[n] = path - - # if tagged, remove the leading keyword - elif record.get("tagged", False): - keyword = next(iter(fields), None) - if keyword: - fields.pop(keyword) - - return fields - - # list input can have records or unions as rows. - # lists which have a consistent record type are - # regular, inconsistent record types irregular. - if type_.startswith("recarray"): - # make sure columns are defined - names = type_.split()[1:] - n_names = len(names) - if n_names < 1: - raise ValueError(f"Missing recarray definition: {type_}") - - # regular tabular/columnar data (1 record type) can be - # defined with a nested record (i.e. explicit) or with - # fields directly inside the recarray (implicit). list - # data for unions/keystrings necessarily comes nested. - - def _is_explicit_record(): - return len(names) == 1 and definition[names[0]][ - "type" - ].startswith("record") - - def _is_implicit_record(): - types = [ - fullname(v["type"]) - for n, v in definition.items() - if n in names - ] - scalar_types = list(SCALAR_TYPES.keys()) - return all(t in scalar_types for t in types) - - if _is_explicit_record(): - name = names[0] - record_type = _map_var(definition[name]) - var_["type"] = List[record_type["type"]] - var_["children"] = {name: record_type} - var_["is_list"] = True - elif _is_implicit_record(): - # record implicitly defined, make it on the fly - name = name_ - fields = _get_record_fields(name) - record_type = Tuple[ - tuple([f["type"] for f in fields.values()]) - ] - record = { - "name": name, - "type": record_type, - "children": fields, - "is_array": False, - "is_record": True, - "is_union": False, - "is_list": False, - "is_variadic": False, - "is_choice": False, - } - var_["type"] = List[record_type] - var_["children"] = {name: record} - var_["is_list"] = True - else: - # irregular recarray, rows can be any of several types - children = {n: _map_var(definition[n]) for n in names} - var_["type"] = List[ - Union[tuple([c["type"] for c in children.values()])] - ] - var_["children"] = children - var_["is_list"] = True - - # now the basic composite types... - # union (product) type, children are choices of records - elif type_.startswith("keystring"): - names = type_.split()[1:] - children = {n: _map_var(definition[n], wrap=True) for n in names} - var_["type"] = Union[tuple([c["type"] for c in children.values()])] - var_["children"] = children - var_["is_union"] = True - - # record (sum) type, children are fields - elif type_.startswith("record"): - name = name_ - fields = _get_record_fields(name) - if len(fields) > 1: - record_type = Tuple[ - tuple([c["type"] for c in fields.values()]) - ] - elif len(fields) == 1: - t = list(fields.values())[0]["type"] - # make sure we don't double-wrap tuples - record_type = t if get_origin(t) is tuple else Tuple[(t,)] - # TODO: if record has 1 field, accept value directly? - var_["type"] = record_type - var_["children"] = fields - var_["is_record"] = True - - # are we wrapping a choice in a union? - # if so, use a literal for the leading - # keyword like tuple (Literal[...], T) - elif wrap: - name = name_ - field = _map_var(var) - fields = {name: field} - field_type = ( - Literal[name] if field["type"] is bool else field["type"] - ) - record_type = ( - Tuple[Literal[name]] - if field["type"] is bool - else Tuple[Literal[name], field["type"]] - ) - fields[name] = {**field, "type": field_type} - var_["type"] = record_type - var_["children"] = fields - var_["is_record"] = True - var_["is_choice"] = True - - # at this point, if it has a shape, it's an array. - # but if it's in a record use a variadic tuple. - elif shape is not None: - if var.get("in_record", False): - if type_ not in SCALAR_TYPES.keys(): - raise TypeError(f"Unsupported repeating type: {type_}") - var_["type"] = Tuple[SCALAR_TYPES[type_], ...] - var_["is_variadic"] = True - elif type_ == "string": - var_["type"] = Tuple[SCALAR_TYPES[type_], ...] - var_["is_variadic"] = True - else: - if type_ not in NP_SCALAR_TYPES.keys(): - raise TypeError(f"Unsupported array type: {type_}") - var_["type"] = NDArray[NP_SCALAR_TYPES[type_]] - var_["is_array"] = True - - # finally a bog standard scalar - else: - # if it's a keyword tag for another - # variable, make it a string literal - tag = type_ == "keyword" and (wrap or var.get("tagged", False)) - var_["type"] = Literal[name_] if tag else SCALAR_TYPES[type_] - - # make optional if needed - if var_.get("optional", True): - var_["type"] = ( - Optional[var_["type"]] - if ( - var_["type"] is not bool - and var_.get("optional", True) - and not var_.get("in_record", False) - and not wrap - ) - else var_["type"] - ) - - # keywords default to False, everything else to None - var_["default"] = var.pop( - "default", False if var_["type"] is bool else None - ) - - # make substitutions from common variables - # and remove backslashes from description - def _map_descr(description: str) -> str: - description = description.replace("\\", "") - _, replace, tail = description.strip().partition("REPLACE") - if replace: - key, _, replacements = tail.strip().partition(" ") - replacements = eval(replacements) - val = common_vars.get(key, None).get("description", "") - if val is None: - raise ValueError(f"Common variable not found: {key}") - if any(replacements): - return val.replace("{#1}", replacements["{#1}"]) - return val - return description - - var_["description"] = _map_descr(var_.get("description", "")) - - # if name is a reserved keyword, - # add trailing underscore to it - var_["name"] = ( - f"{var_['name']}_" if var_["name"] in kwlist else var_["name"] - ) - - return var_ - - def _qualify(var: dict) -> dict: - """ - Recursively convert the variable's type to a fully qualified string. - """ - - var["type"] = fullname(var["type"]) - children = var.get("children", dict()) - if any(children): - var["children"] = {n: _qualify(c) for n, c in children.items()} - return var - - def _variables(vars: dict) -> dict: - return { - name: _qualify(_map_var(var)) - for name, var in vars.items() - # filter components of composites - # since we've inflated the parent - # types in the hierarchy already - if not var.get("in_record", False) - } - - def _dfn(vars: dict, meta: list) -> list: - """ - Currently, generated classes have a `.dfn` property that - reproduces the corresponding DFN sans a few attributes. - """ - - def _var_dfn(var: dict) -> List[str]: - exclude = ["longname", "description"] - return [ - " ".join([k, v]) for k, v in var.items() if k not in exclude - ] - - return [["header"] + [attr for attr in meta]] + [ - _var_dfn(var) for var in vars.values() - ] - - return { - "component": component, - "subcomponent": subcomponent, - "variables": _variables(definition), - "dfn": _dfn(definition, metadata), - } - - -@pytest.mark.parametrize( - "dfn, n_flat, n_nested", [("gwf-ic", 2, 2), ("prt-prp", 40, 18)] -) -def test_get_template_context(dfn, n_flat, n_nested): - component, subcomponent = dfn.split("-") - - with open(DFNS_PATH / "common.dfn") as f: - common_vars, _ = load_dfn(f) - - with open(DFNS_PATH / f"{dfn}.dfn") as f: - variables, metadata = load_dfn(f) - - context = get_template_context( - component, subcomponent, common_vars, variables, metadata - ) - assert context["component"] == component - assert context["subcomponent"] == subcomponent - assert len(context["variables"]) == n_nested - assert len(context["dfn"]) == n_flat + 1 # +1 for metadata - - -from jinja2 import Environment, PackageLoader - -TEMPLATE_ENV = Environment(loader=PackageLoader("flopy", "mf6/templates/")) - - -class ComponentType(Enum): - Package = "package" - Simulation = "simulation" - - -def get_component_type(component, subcomponent) -> ComponentType: - if component == "sim" and subcomponent == "nam": - return ComponentType.Simulation - else: - return ComponentType.Package - - -@pytest.mark.parametrize( - "dfn", - [ - # packages - "gwf-ic", - "prt-prp", - "gwe-ctp", - "gwe-cnd", - "gwe-esl", - "gwf-dis", - "prt-mip", - "prt-oc", - "gwf-oc", - # models - "gwf-nam", - "gwt-nam", - "gwe-nam", - "prt-nam", - # solutions - "sln-ims", - "sln-ems", - # simulation - "sim-nam", - ], -) -def test_render_template(dfn, function_tmpdir): - component, subcomponent = dfn.split("-") - comp_name = f"{component}{subcomponent}" - comp_type = get_component_type(component, subcomponent).value - template = TEMPLATE_ENV.get_template(f"{comp_type}.jinja") - - with open(DFNS_PATH / "common.dfn") as f: - common_vars, _ = load_dfn(f) - - with open(DFNS_PATH / f"{dfn}.dfn", "r") as f: - variables, metadata = load_dfn(f) - - context = get_template_context( - component, subcomponent, common_vars, variables, metadata - ) - source = template.render(**context) - source_path = function_tmpdir / f"{comp_name}.py" - with open(source_path, "w") as f: - f.write(source) - run_cmd("ruff", "format", source_path, verbose=True) diff --git a/autotest/test_createpackages.py b/autotest/test_createpackages.py new file mode 100644 index 0000000000..0b0da07db1 --- /dev/null +++ b/autotest/test_createpackages.py @@ -0,0 +1,70 @@ +import pytest +from modflow_devtools.misc import run_cmd + +from autotest.conftest import get_project_root_path +from flopy.mf6.utils.createpackages import ( + TEMPLATE_ENV, + ContextType, + get_template_context, +) +from flopy.mf6.utils.dfn import load_dfn + +PROJ_ROOT = get_project_root_path() +DFNS_PATH = PROJ_ROOT / "flopy" / "mf6" / "data" / "dfn" +DFNS = [ + dfn + for dfn in DFNS_PATH.glob("*.dfn") + if dfn.stem not in ["common", "flopy"] +] + + +# only test packages for which we know the +# expected consolidated number of variables +@pytest.mark.parametrize( + "dfn, n_flat, n_nested", [("gwf-ic", 2, 2), ("prt-prp", 40, 18)] +) +def test_get_template_context(dfn, n_flat, n_nested): + component, subcomponent = dfn.split("-") + + with open(DFNS_PATH / "common.dfn") as f: + common_vars, _ = load_dfn(f) + + with open(DFNS_PATH / "flopy.dfn") as f: + flopy_vars, _ = load_dfn(f) + + with open(DFNS_PATH / f"{dfn}.dfn") as f: + variables, metadata = load_dfn(f) + + context = get_template_context( + component, subcomponent, common_vars, flopy_vars, variables, metadata + ) + assert context["component"] == component + assert context["subcomponent"] == subcomponent + assert len(context["variables"]) == n_nested + assert len(context["dfn"]) == n_flat + 1 # +1 for metadata + + +@pytest.mark.parametrize("dfn", [dfn.stem for dfn in DFNS]) +def test_render_template(dfn, function_tmpdir): + component, subcomponent = dfn.split("-") + context_name = f"{component}{subcomponent}" + context_type = ContextType.from_pair(component, subcomponent).value + template = TEMPLATE_ENV.get_template(f"{context_type}.jinja") + + with open(DFNS_PATH / "common.dfn") as f: + common_vars, _ = load_dfn(f) + + with open(DFNS_PATH / "flopy.dfn") as f: + flopy_vars, _ = load_dfn(f) + + with open(DFNS_PATH / f"{dfn}.dfn", "r") as f: + variables, metadata = load_dfn(f) + + context = get_template_context( + component, subcomponent, common_vars, flopy_vars, variables, metadata + ) + source = template.render(**context) + source_path = function_tmpdir / f"{context_name}.py" + with open(source_path, "w") as f: + f.write(source) + run_cmd("ruff", "format", source_path, verbose=True) diff --git a/autotest/test_dfn.py b/autotest/test_dfn.py new file mode 100644 index 0000000000..0151341a9d --- /dev/null +++ b/autotest/test_dfn.py @@ -0,0 +1,15 @@ +import pytest + +from autotest.conftest import get_project_root_path +from flopy.mf6.utils.dfn import load_dfn + +PROJ_ROOT = get_project_root_path() +DFNS_PATH = PROJ_ROOT / "flopy" / "mf6" / "data" / "dfn" +DFNS = [dfn.name for dfn in DFNS_PATH.glob("*.dfn")] + + +@pytest.mark.parametrize("dfn", DFNS) +def test_load_dfn(dfn): + dfn_path = DFNS_PATH / dfn + with open(dfn_path, "r") as f: + vars, meta = load_dfn(f) diff --git a/flopy/mf6/templates/model.jinja b/flopy/mf6/templates/model.jinja new file mode 100644 index 0000000000..afcb06c16b --- /dev/null +++ b/flopy/mf6/templates/model.jinja @@ -0,0 +1,141 @@ +# autogenerated file, do not modify +from .. import mfmodel +from ..data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenerator + + +class Modflow{{ component.title() }}(mfmodel.MFModel): + """ + Modflow{{ component }} defines a {{ component }} model + + Parameters + ---------- + modelname : string + name of the model + model_nam_file : string + relative path to the model name file from model working folder + version : string + version of modflow + exe_name : string + model executable name + model_ws : string + model working folder path + sim : MFSimulation + Simulation that this model is a part of. Model is automatically + added to simulation when it is initialized. + list : string + * list (string) is name of the listing file to create for this {{ component.upper() }} + model. If not specified, then the name of the list file will be the + basename of the {{ component.upper() }} model name file and the '.lst' extension. For + example, if the {{ component.upper() }} name file is called "my.model.nam" then the list + file will be called "my.model.lst". + print_input : boolean + * print_input (boolean) keyword to indicate that the list of all model + stress package information will be written to the listing file + immediately after it is read. + print_flows : boolean + * print_flows (boolean) keyword to indicate that the list of all model + package flow rates will be printed to the listing file for every + stress period time step in which "BUDGET PRINT" is specified in + Output Control. If there is no Output Control option and + "PRINT_FLOWS" is specified, then flow rates are printed for the last + time step of each stress period. + save_flows : boolean + * save_flows (boolean) keyword to indicate that all model package flow + terms will be written to the file specified with "BUDGET FILEOUT" in + Output Control. + newtonoptions : [under_relaxation] + * under_relaxation (string) keyword that indicates whether the + groundwater head in a cell will be under-relaxed when water levels + fall below the bottom of the model below any given cell. By default, + Newton-Raphson UNDER_RELAXATION is not applied. + packages : [ftype, fname, pname] + * ftype (string) is the file type, which must be one of the following + character values shown in table ref{table:ftype-{{ component }}}. Ftype may be + entered in any combination of uppercase and lowercase. + * fname (string) is the name of the file containing the package input. + The path to the file should be included if the file is not located in + the folder where the program was run. + * pname (string) is the user-defined name for the package. PNAME is + restricted to 16 characters. No spaces are allowed in PNAME. PNAME + character values are read and stored by the program for stress + packages only. These names may be useful for labeling purposes when + multiple stress packages of the same type are located within a single + {{ component.upper() }} Model. If PNAME is specified for a stress package, then PNAME + will be used in the flow budget table in the listing file; it will + also be used for the text entry in the cell-by-cell budget file. + PNAME is case insensitive and is stored in all upper case letters. + + Methods + ------- + load : (simulation : MFSimulationData, model_name : string, + namfile : string, version : string, exe_name : string, + model_ws : string, strict : boolean) : MFSimulation + a class method that loads a model from files + """ + + model_type = "{{ component }}" + + def __init__( + self, + simulation, + modelname="model", + model_nam_file=None, + version="mf6", + exe_name="mf6", + model_rel_path=".", + list=None, + print_input=None, + print_flows=None, + save_flows=None, + newtonoptions=None, + **kwargs, + ): + super().__init__( + simulation, + model_type="{{ component }}6", + modelname=modelname, + model_nam_file=model_nam_file, + version=version, + exe_name=exe_name, + model_rel_path=model_rel_path, + **kwargs, + ) + + self.name_file.list.set_data(list) + self.name_file.print_input.set_data(print_input) + self.name_file.print_flows.set_data(print_flows) + self.name_file.save_flows.set_data(save_flows) + self.name_file.newtonoptions.set_data(newtonoptions) + + self.list = self.name_file.list + self.print_input = self.name_file.print_input + self.print_flows = self.name_file.print_flows + self.save_flows = self.name_file.save_flows + self.newtonoptions = self.name_file.newtonoptions + + @classmethod + def load( + cls, + simulation, + structure, + modelname="NewModel", + model_nam_file="modflowtest.nam", + version="mf6", + exe_name="mf6", + strict=True, + model_rel_path=".", + load_only=None, + ): + return mfmodel.MFModel.load_base( + cls, + simulation, + structure, + modelname, + model_nam_file, + "{{ component }}6", + version, + exe_name, + strict, + model_rel_path, + load_only, + ) diff --git a/flopy/mf6/utils/codegen.py b/flopy/mf6/utils/codegen.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index e1a57fb094..d72a968b5a 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -85,9 +85,27 @@ import os import textwrap from enum import Enum +from keyword import kwlist +from pathlib import Path +from typing import ( + Dict, + ForwardRef, + List, + Literal, + Optional, + Tuple, + Union, + get_args, + get_origin, +) + +import numpy as np +from jinja2 import Environment, PackageLoader +from numpy.typing import NDArray # keep below as absolute imports from flopy.mf6.data import mfdatautil, mfstructure +from flopy.mf6.utils.dfn import Definition, load_dfn from flopy.utils import datautil @@ -1132,5 +1150,474 @@ def create_packages(): init_file.close() +DFNS_PATH = Path(__file__).parents[1] / "data" / "dfn" +SRCS_PATH = Path(__file__).parents[1] / "modflow" +SCALAR_TYPES = { + "keyword": bool, + "integer": int, + "double precision": float, + "string": str, +} +NP_SCALAR_TYPES = { + "keyword": np.bool_, + "integer": np.int_, + "double precision": np.float64, + "string": np.str_, +} +TEMPLATE_ENV = Environment(loader=PackageLoader("flopy", "mf6/templates/")) + + +class ContextType(Enum): + Model = "model" + Package = "package" + Simulation = "simulation" + + @classmethod + def from_pair(cls, component, subcomponent) -> "ContextType": + if component == "sim" and subcomponent == "nam": + return ContextType.Simulation + elif subcomponent == "nam": + return ContextType.Model + else: + return ContextType.Package + + +def fullname(t: type) -> str: + """Convert a type to a name suitable for templating.""" + origin = get_origin(t) + args = get_args(t) + if origin is Literal: + args = ['"' + a + '"' for a in args] + return f"{Literal.__name__}[{', '.join(args)}]" + elif origin is Union: + if len(args) == 2 and args[1] is type(None): + return f"{Optional.__name__}[{fullname(args[0])}]" + return f"{Union.__name__}[{', '.join([fullname(a) for a in args])}]" + elif origin is tuple: + return f"{Tuple.__name__}[{', '.join([fullname(a) for a in args])}]" + elif origin is list: + return f"{List.__name__}[{', '.join([fullname(a) for a in args])}]" + elif origin is np.ndarray: + return f"NDArray[np.{fullname(args[1].__args__[0])}]" + elif origin is np.dtype: + return str(t) + elif isinstance(t, ForwardRef): + return t.__forward_arg__ + elif t is Ellipsis: + return "..." + elif isinstance(t, type): + return t.__qualname__ + else: + return str(t) + + +def get_template_context( + component: str, + subcomponent: str, + common_vars: Definition, + flopy_vars: Definition, + definition: Definition, + metadata: List[str], +) -> dict: + """ + Convert an input definition to a template rendering context. + + TODO: pull out a class for the input definition, and expose + this as an instance method? + """ + + def _convert(var: dict, wrap: bool = False) -> dict: + """ + Transform a variable from its original representation in + an input definition to a form suitable for type hints and + and docstrings. + + This involves expanding nested type hierarchies, converting + input types to equivalent Python primitives and composites, + and various other shaping. + + Notes + ----- + If a `default_value` is not provided, keywords are `False` + by default. Everything else is `None` by default. + + If `wrap` is true, scalars will be wrapped as records with + keywords represented as string literals. This is useful for + unions, to distinguish between choices having the same type. + """ + var_ = { + **var, + # some flags the template uses for formatting. + # these are ofc derivable in Python but Jinja + # doesn't allow arbitrary expressions, and it + # doesn't seem to have `subclass`-ish filters. + # (we convert the variable type to string too + # before returning, for the same reason.) + "is_array": False, + "is_list": False, + "is_record": False, + "is_union": False, + "is_variadic": False, + "is_choice": False, + } + name_ = var["name"] + type_ = var["type"] + shape = var.get("shape", None) + shape = None if shape == "" else shape + + # utilities for generating records + # as named tuples. + + def _get_record_fields(name: str) -> dict: + """ + Call `_map_var` recursively on each field + of the record variable with the given name. + + Notes + ----- + This function is provided because records + need extra processing; we remove keywords + and 'filein'/'fileout', which are details + of the mf6io format, not of python/flopy. + """ + record = definition[name] + names = record["type"].split()[1:] + fields = { + n: {**_convert(field), "optional": field.get("optional", True)} + for n, field in definition.items() + if n in names + } + field_names = list(fields.keys()) + + # if the record represents a file... + if "file" in name: + # remove filein/fileout + for term in ["filein", "fileout"]: + if term in field_names: + fields.pop(term) + + # remove leading keyword + keyword = next(iter(fields), None) + if keyword: + fields.pop(keyword) + + # set the type + n = list(fields.keys())[0] + path = fields[n] + path["type"] = os.PathLike + fields[n] = path + + # if tagged, remove the leading keyword + elif record.get("tagged", False): + keyword = next(iter(fields), None) + if keyword: + fields.pop(keyword) + + return fields + + # list input can have records or unions as rows. + # lists which have a consistent record type are + # regular, inconsistent record types irregular. + if type_.startswith("recarray"): + # make sure columns are defined + names = type_.split()[1:] + n_names = len(names) + if n_names < 1: + raise ValueError(f"Missing recarray definition: {type_}") + + # regular tabular/columnar data (1 record type) can be + # defined with a nested record (i.e. explicit) or with + # fields directly inside the recarray (implicit). list + # data for unions/keystrings necessarily comes nested. + + def _is_explicit_record(): + return len(names) == 1 and definition[names[0]][ + "type" + ].startswith("record") + + def _is_implicit_record(): + types = [ + fullname(v["type"]) + for n, v in definition.items() + if n in names + ] + scalar_types = list(SCALAR_TYPES.keys()) + return all(t in scalar_types for t in types) + + if _is_explicit_record(): + name = names[0] + record_type = _convert(definition[name]) + var_["type"] = List[record_type["type"]] + var_["children"] = {name: record_type} + var_["is_list"] = True + elif _is_implicit_record(): + # record implicitly defined, make it on the fly + name = name_ + fields = _get_record_fields(name) + record_type = Tuple[ + tuple([f["type"] for f in fields.values()]) + ] + record = { + "name": name, + "type": record_type, + "children": fields, + "is_array": False, + "is_record": True, + "is_union": False, + "is_list": False, + "is_variadic": False, + "is_choice": False, + } + var_["type"] = List[record_type] + var_["children"] = {name: record} + var_["is_list"] = True + else: + # irregular recarray, rows can be any of several types + children = {n: _convert(definition[n]) for n in names} + var_["type"] = List[ + Union[tuple([c["type"] for c in children.values()])] + ] + var_["children"] = children + var_["is_list"] = True + + # now the basic composite types... + # union (product) type, children are choices of records + elif type_.startswith("keystring"): + names = type_.split()[1:] + children = {n: _convert(definition[n], wrap=True) for n in names} + var_["type"] = Union[tuple([c["type"] for c in children.values()])] + var_["children"] = children + var_["is_union"] = True + + # record (sum) type, children are fields + elif type_.startswith("record"): + name = name_ + fields = _get_record_fields(name) + if len(fields) > 1: + record_type = Tuple[ + tuple([c["type"] for c in fields.values()]) + ] + elif len(fields) == 1: + t = list(fields.values())[0]["type"] + # make sure we don't double-wrap tuples + record_type = t if get_origin(t) is tuple else Tuple[(t,)] + # TODO: if record has 1 field, accept value directly? + var_["type"] = record_type + var_["children"] = fields + var_["is_record"] = True + + # are we wrapping a choice in a union? + # if so, use a literal for the leading + # keyword like tuple (Literal[...], T) + elif wrap: + name = name_ + field = _convert(var) + fields = {name: field} + field_type = ( + Literal[name] if field["type"] is bool else field["type"] + ) + record_type = ( + Tuple[Literal[name]] + if field["type"] is bool + else Tuple[Literal[name], field["type"]] + ) + fields[name] = {**field, "type": field_type} + var_["type"] = record_type + var_["children"] = fields + var_["is_record"] = True + var_["is_choice"] = True + + # at this point, if it has a shape, it's an array. + # but if it's in a record use a variadic tuple. + elif shape is not None: + if var.get("in_record", False): + if type_ not in SCALAR_TYPES.keys(): + raise TypeError(f"Unsupported repeating type: {type_}") + var_["type"] = Tuple[SCALAR_TYPES[type_], ...] + var_["is_variadic"] = True + elif type_ == "string": + var_["type"] = Tuple[SCALAR_TYPES[type_], ...] + var_["is_variadic"] = True + else: + if type_ not in NP_SCALAR_TYPES.keys(): + raise TypeError(f"Unsupported array type: {type_}") + var_["type"] = NDArray[NP_SCALAR_TYPES[type_]] + var_["is_array"] = True + + # finally a bog standard scalar + else: + # if it's a keyword tag for another + # variable, make it a string literal + tag = type_ == "keyword" and (wrap or var.get("tagged", False)) + var_["type"] = Literal[name_] if tag else SCALAR_TYPES[type_] + + # make optional if needed + if var_.get("optional", True): + var_["type"] = ( + Optional[var_["type"]] + if ( + var_["type"] is not bool + and var_.get("optional", True) + and not var_.get("in_record", False) + and not wrap + ) + else var_["type"] + ) + + # keywords default to False, everything else to None + var_["default"] = var.pop( + "default", False if var_["type"] is bool else None + ) + + # make substitutions from common variables + # and remove backslashes from description + def _map_descr(description: str) -> str: + description = description.replace("\\", "") + _, replace, tail = description.strip().partition("REPLACE") + if replace: + key, _, replacements = tail.strip().partition(" ") + replacements = eval(replacements) + val = common_vars.get(key, None).get("description", "") + if val is None: + raise ValueError(f"Common variable not found: {key}") + if any(replacements): + return val.replace("{#1}", replacements["{#1}"]) + return val + return description + + var_["description"] = _map_descr(var_.get("description", "")) + + # if name is a reserved keyword, add a trailing underscore to it + var_["name"] = ( + f"{var_['name']}_" if var_["name"] in kwlist else var_["name"] + ) + + return var_ + + def _qualify(var: dict) -> dict: + """ + Recursively convert the variable's type to a fully qualified string. + + Notes + ----- + Separate function operating on the entire variable (rather than the + type alone) because we want to pass typed definitions around until + conversion just before templating. + """ + + var["type"] = fullname(var["type"]) + children = var.get("children", dict()) + if any(children): + var["children"] = {n: _qualify(c) for n, c in children.items()} + return var + + def _variables(vars: dict) -> dict: + """Get the class' member variables.""" + return { + name: _qualify(_convert(var)) + for name, var in vars.items() + # filter components of composites + # since we've inflated the parent + # types in the hierarchy already + if not var.get("in_record", False) + } + + def _dfn(vars: dict, meta: list) -> list: + """ + Get a list of the class' original definition attributes. + + Notes + ----- + Currently, generated classes have a `.dfn` property that + reproduces the corresponding DFN sans a few attributes. + This represents the DFN in raw form, before adapting to + Python, consolidating nested types, etc. + """ + + def _var_dfn(var: dict) -> List[str]: + exclude = ["longname", "description"] + return [ + " ".join([k, v]) for k, v in var.items() if k not in exclude + ] + + return [["header"] + [attr for attr in meta]] + [ + _var_dfn(var) for var in vars.values() + ] + + return { + "component": component, + "subcomponent": subcomponent, + "variables": _variables(definition), + "dfn": _dfn(definition, metadata), + } + + +def get_source_path(component, subcomponent): + def _name(): + _case = (component, subcomponent) + if _case == ("sim", "nam"): + return "simulation" + elif _case == ("sim", "tdis"): + return "tdis" + elif component == "sln": + return subcomponent + return f"{component}{subcomponent}" + + return SRCS_PATH / f"mf{_name()}.py" + + +def generate_component(dfn_path): + comp, sub = dfn_path.stem.split("-") + py_path = get_source_path(comp, sub) + is_model = comp != "sim" and sub == "nam" + + with open(DFNS_PATH / "common.dfn") as f: + common_vars, _ = load_dfn(f) + + with open(DFNS_PATH / "flopy.dfn") as f: + flopy_vars, _ = load_dfn(f) + + with open(dfn_path, "r") as f: + vars, meta = load_dfn(f) + + with open(py_path, "w") as f: + context = get_template_context( + component=comp, + subcomponent=sub, + common_vars=common_vars, + flopy_vars=flopy_vars, + variables=vars, + metadata=meta, + ) + template = TEMPLATE_ENV.get_template( + f"{ContextType.from_pair(comp, sub)}.jinja" + ) + source = template.render(**context) + f.write(source) + + if is_model: + py_path = SRCS_PATH / f"mf{comp}.py" + with open(py_path, "w") as f: + context = get_template_context( + component=comp, + subcomponent=sub, + common_vars=common_vars, + flopy_vars=flopy_vars, + variables=vars, + metadata=meta, + ) + template = TEMPLATE_ENV.get_template( + f"{ContextType.from_pair(comp, sub)}.jinja" + ) + source = template.render(**context) + f.write(source) + + +def generate_components(): + for dfn_path in DFNS_PATH.glob("*.dfn"): + generate_component(dfn_path) + + if __name__ == "__main__": create_packages() + # generate_components diff --git a/flopy/mf6/utils/dfn.py b/flopy/mf6/utils/dfn.py new file mode 100644 index 0000000000..12bf1506bd --- /dev/null +++ b/flopy/mf6/utils/dfn.py @@ -0,0 +1,46 @@ +from typing import Dict, List, Tuple, Union + +Scalar = Union[bool, int, float, str] +Definition = Dict[str, Dict[str, Scalar]] + + +def load_dfn(f) -> Tuple[Definition, List[str]]: + """ + Load an input definition file. Returns a tuple containing + a dictionary variables and a list of metadata attributes. + """ + meta = list() + vars = dict() + var = dict() + + for line in f: + # remove whitespace/etc from the line + line = line.strip() + + # record flopy metadata attributes but + # skip all other comment lines + if line.startswith("#"): + _, sep, tail = line.partition("flopy") + if sep == "flopy": + meta.append(tail.strip()) + continue + + # if we hit a newline and the parameter dict + # is nonempty, we've reached the end of its + # block of attributes + if not any(line): + if any(var): + vars[var["name"]] = var + var = dict() + continue + + # split the attribute's key and value and + # store it in the parameter dictionary + key, _, value = line.partition(" ") + var[key] = value + + # add the final parameter + if any(var): + vars[var["name"]] = var + + return vars, meta From a5a580c39a687b909e546f537e0e1c27f3a3d9f2 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 25 Sep 2024 20:32:04 -0400 Subject: [PATCH 08/46] cleanup, add jinja to pyproject.toml --- flopy/mf6/utils/codegen.py | 0 pyproject.toml | 1 + 2 files changed, 1 insertion(+) delete mode 100644 flopy/mf6/utils/codegen.py diff --git a/flopy/mf6/utils/codegen.py b/flopy/mf6/utils/codegen.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/pyproject.toml b/pyproject.toml index f18d5cd164..eed4525e85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ test = [ "coverage", "flaky", "filelock", + "Jinja2", "jupyter", "jupyter_client >=8.4.0", # avoid datetime.utcnow() deprecation warning "jupytext", From 5a8b1be75f1fc78bac6f6b09ae6e15312f71516e Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 25 Sep 2024 20:40:58 -0400 Subject: [PATCH 09/46] cleanup --- autotest/test_createpackages.py | 4 ++-- flopy/mf6/utils/createpackages.py | 14 +++++++------- pyproject.toml | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/autotest/test_createpackages.py b/autotest/test_createpackages.py index 0b0da07db1..4a7eacbfd0 100644 --- a/autotest/test_createpackages.py +++ b/autotest/test_createpackages.py @@ -4,7 +4,7 @@ from autotest.conftest import get_project_root_path from flopy.mf6.utils.createpackages import ( TEMPLATE_ENV, - ContextType, + TemplateType, get_template_context, ) from flopy.mf6.utils.dfn import load_dfn @@ -48,7 +48,7 @@ def test_get_template_context(dfn, n_flat, n_nested): def test_render_template(dfn, function_tmpdir): component, subcomponent = dfn.split("-") context_name = f"{component}{subcomponent}" - context_type = ContextType.from_pair(component, subcomponent).value + context_type = TemplateType.from_pair(component, subcomponent).value template = TEMPLATE_ENV.get_template(f"{context_type}.jinja") with open(DFNS_PATH / "common.dfn") as f: diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index d72a968b5a..b8e6d55915 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -1167,19 +1167,19 @@ def create_packages(): TEMPLATE_ENV = Environment(loader=PackageLoader("flopy", "mf6/templates/")) -class ContextType(Enum): +class TemplateType(Enum): Model = "model" Package = "package" Simulation = "simulation" @classmethod - def from_pair(cls, component, subcomponent) -> "ContextType": + def from_pair(cls, component, subcomponent) -> "TemplateType": if component == "sim" and subcomponent == "nam": - return ContextType.Simulation + return TemplateType.Simulation elif subcomponent == "nam": - return ContextType.Model + return TemplateType.Model else: - return ContextType.Package + return TemplateType.Package def fullname(t: type) -> str: @@ -1590,7 +1590,7 @@ def generate_component(dfn_path): metadata=meta, ) template = TEMPLATE_ENV.get_template( - f"{ContextType.from_pair(comp, sub)}.jinja" + f"{TemplateType.from_pair(comp, sub)}.jinja" ) source = template.render(**context) f.write(source) @@ -1607,7 +1607,7 @@ def generate_component(dfn_path): metadata=meta, ) template = TEMPLATE_ENV.get_template( - f"{ContextType.from_pair(comp, sub)}.jinja" + f"{TemplateType.from_pair(comp, sub)}.jinja" ) source = template.render(**context) f.write(source) diff --git a/pyproject.toml b/pyproject.toml index eed4525e85..0ab15b6415 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,9 +29,10 @@ classifiers = [ ] requires-python = ">=3.8" dependencies = [ + "Jinja2", "numpy>=1.20.3", "matplotlib >=1.4.0", - "pandas >=2.0.0" + "pandas >=2.0.0", ] dynamic = ["version", "readme"] @@ -46,7 +47,6 @@ test = [ "coverage", "flaky", "filelock", - "Jinja2", "jupyter", "jupyter_client >=8.4.0", # avoid datetime.utcnow() deprecation warning "jupytext", From 47f04bd7ba432decc93766a804daa816b06c4ec2 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 26 Sep 2024 10:36:12 -0400 Subject: [PATCH 10/46] exg handling, more reorg, misc fixes, todo: subpackages --- autotest/test_createpackages.py | 12 ++- flopy/mf6/templates/attrs.jinja | 2 +- flopy/mf6/templates/docstring.jinja | 8 +- flopy/mf6/templates/init.jinja | 12 ++- flopy/mf6/templates/model.jinja | 4 +- flopy/mf6/templates/package.jinja | 3 +- flopy/mf6/utils/createpackages.py | 137 ++++++++++++++++++++-------- pyproject.toml | 2 +- 8 files changed, 129 insertions(+), 51 deletions(-) diff --git a/autotest/test_createpackages.py b/autotest/test_createpackages.py index 4a7eacbfd0..8fb191123e 100644 --- a/autotest/test_createpackages.py +++ b/autotest/test_createpackages.py @@ -5,6 +5,8 @@ from flopy.mf6.utils.createpackages import ( TEMPLATE_ENV, TemplateType, + generate_components, + get_src_name, get_template_context, ) from flopy.mf6.utils.dfn import load_dfn @@ -48,8 +50,8 @@ def test_get_template_context(dfn, n_flat, n_nested): def test_render_template(dfn, function_tmpdir): component, subcomponent = dfn.split("-") context_name = f"{component}{subcomponent}" - context_type = TemplateType.from_pair(component, subcomponent).value - template = TEMPLATE_ENV.get_template(f"{context_type}.jinja") + template_type = TemplateType.from_pair(component, subcomponent).value + template = TEMPLATE_ENV.get_template(f"{template_type}.jinja") with open(DFNS_PATH / "common.dfn") as f: common_vars, _ = load_dfn(f) @@ -64,7 +66,11 @@ def test_render_template(dfn, function_tmpdir): component, subcomponent, common_vars, flopy_vars, variables, metadata ) source = template.render(**context) - source_path = function_tmpdir / f"{context_name}.py" + source_path = function_tmpdir / get_src_name(component, subcomponent) with open(source_path, "w") as f: f.write(source) run_cmd("ruff", "format", source_path, verbose=True) + + +def test_generate_components(function_tmpdir): + generate_components(function_tmpdir, verbose=True) diff --git a/flopy/mf6/templates/attrs.jinja b/flopy/mf6/templates/attrs.jinja index 3573443b5d..2b77e65abb 100644 --- a/flopy/mf6/templates/attrs.jinja +++ b/flopy/mf6/templates/attrs.jinja @@ -1,6 +1,6 @@ {# Class attributes for a component class. #} {% for name, var in variables.items() if var.type %} - {%- if var.is_list or var.is_record %} + {%- if var.is_iterable or var.is_list or var.is_record %} {{ var.name }} = ListTemplateGenerator(("{{ component }}6", "{{ subcomponent }}", "{{ var.block }}", "{{ var.name }}")) {%- elif var.is_array %} {{ var.name }} = ArrayTemplateGenerator(("{{ component }}6", "{{ subcomponent }}", "{{ var.block }}", "{{ var.name }}")) diff --git a/flopy/mf6/templates/docstring.jinja b/flopy/mf6/templates/docstring.jinja index b437e90096..687fb29a43 100644 --- a/flopy/mf6/templates/docstring.jinja +++ b/flopy/mf6/templates/docstring.jinja @@ -1,11 +1,11 @@ {# A component class' docstring. #} - Modflow{{ component.title() }}{{ subcomponent }} defines a {{ subcomponent }} package within a {{ component }}6 model. + Modflow{{ title }} defines a {{ subcomponent }} package within a {{ parent.name }}. Parameters ---------- - model : MFModel - Model that this package is a part of. Package is automatically - added to model when it is initialized. + {{ parent.name }} : {{ parent.type }} + {{ parent.name.title() }} that this package is a part of. Package is automatically + added to {{ parent.name }} when it is initialized. loading_package : bool Do not set this variable. It is intended for debugging and internal diff --git a/flopy/mf6/templates/init.jinja b/flopy/mf6/templates/init.jinja index 94f646d066..a1aa73c189 100644 --- a/flopy/mf6/templates/init.jinja +++ b/flopy/mf6/templates/init.jinja @@ -1,6 +1,6 @@ def __init__( self, - model: MFModel, + {{ parent.name }}: {{ parent.type }}, loading_package: bool = False, {%- for name, var in variables.items() %} {{ name }}: {{ var.type }} = {{ var.default }}, @@ -10,11 +10,17 @@ def __init__( **kwargs, ): super().__init__( - model, "{{ subcomponent }}", filename, pname, loading_package, **kwargs + {{ parent.name }}, "{{ subcomponent }}", filename, pname, loading_package, **kwargs ) # set up variables - {%- for name, var in variables.items() %} + {%- for name, var in variables.items() if not var.set_self %} self.{{ name }} = self.build_mfdata("{{ name }}", {{ name }}) {%- endfor %} + {%- for name, var in variables.items() if var.set_self %} + self.{{ name }} = {{ name }} + {%- endfor %} + {%- if component == "exg" %} + simulation.register_exchange_file(self) + {% endif -%} self._init_complete = True \ No newline at end of file diff --git a/flopy/mf6/templates/model.jinja b/flopy/mf6/templates/model.jinja index afcb06c16b..12350bac3d 100644 --- a/flopy/mf6/templates/model.jinja +++ b/flopy/mf6/templates/model.jinja @@ -3,9 +3,9 @@ from .. import mfmodel from ..data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenerator -class Modflow{{ component.title() }}(mfmodel.MFModel): +class Modflow{{ title }}(mfmodel.MFModel): """ - Modflow{{ component }} defines a {{ component }} model + Modflow{{ title }} defines a {{ component }} model Parameters ---------- diff --git a/flopy/mf6/templates/package.jinja b/flopy/mf6/templates/package.jinja index 30d22df04f..4297f995ab 100644 --- a/flopy/mf6/templates/package.jinja +++ b/flopy/mf6/templates/package.jinja @@ -7,9 +7,10 @@ from numpy.typing import NDArray from .. import mfpackage from ..data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenerator from flopy.mf6.mfmodel import MFModel +from flopy.mf6 import MFSimulation -class Modflow{{ component.title() }}{{ subcomponent }}(mfpackage.MFPackage): +class Modflow{{ title }}(mfpackage.MFPackage): """{% include "docstring.jinja" %}""" {% include "attrs.jinja" %} diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index b8e6d55915..7a43383f48 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -81,6 +81,7 @@ """ +import collections import datetime import os import textwrap @@ -90,6 +91,7 @@ from typing import ( Dict, ForwardRef, + Iterable, List, Literal, Optional, @@ -101,6 +103,7 @@ import numpy as np from jinja2 import Environment, PackageLoader +from modflow_devtools.misc import run_cmd from numpy.typing import NDArray # keep below as absolute imports @@ -1195,6 +1198,8 @@ def fullname(t: type) -> str: return f"{Union.__name__}[{', '.join([fullname(a) for a in args])}]" elif origin is tuple: return f"{Tuple.__name__}[{', '.join([fullname(a) for a in args])}]" + elif origin is collections.abc.Iterable: + return f"{Iterable.__name__}[{', '.join([fullname(a) for a in args])}]" elif origin is list: return f"{List.__name__}[{', '.join([fullname(a) for a in args])}]" elif origin is np.ndarray: @@ -1226,6 +1231,20 @@ def get_template_context( this as an instance method? """ + def _title(comp, sub): + if comp == "sln": + return sub.title() + elif component == "exg": + return sub.title() + return f"{comp.title()}{sub}" + + def _parent(comp, sub): + if comp in ["exg", "sln"]: + return {"name": "simulation", "type": "MFSimulation"} + if comp in ["utl"]: + return {"name": "package", "type": "MFPackage"} + return {"name": "model", "type": "MFModel"} + def _convert(var: dict, wrap: bool = False) -> dict: """ Transform a variable from its original representation in @@ -1257,7 +1276,7 @@ def _convert(var: dict, wrap: bool = False) -> dict: "is_list": False, "is_record": False, "is_union": False, - "is_variadic": False, + "is_iterable": False, "is_choice": False, } name_ = var["name"] @@ -1365,7 +1384,7 @@ def _is_implicit_record(): "is_record": True, "is_union": False, "is_list": False, - "is_variadic": False, + "is_iterable": False, "is_choice": False, } var_["type"] = List[record_type] @@ -1430,14 +1449,14 @@ def _is_implicit_record(): # at this point, if it has a shape, it's an array. # but if it's in a record use a variadic tuple. elif shape is not None: + scalars = list(SCALAR_TYPES.keys()) if var.get("in_record", False): - if type_ not in SCALAR_TYPES.keys(): + if type_ not in scalars: raise TypeError(f"Unsupported repeating type: {type_}") var_["type"] = Tuple[SCALAR_TYPES[type_], ...] - var_["is_variadic"] = True - elif type_ == "string": - var_["type"] = Tuple[SCALAR_TYPES[type_], ...] - var_["is_variadic"] = True + elif type_ in scalars: + var_["type"] = Iterable[SCALAR_TYPES[type_]] + var_["is_iterable"] = True else: if type_ not in NP_SCALAR_TYPES.keys(): raise TypeError(f"Unsupported array type: {type_}") @@ -1464,10 +1483,14 @@ def _is_implicit_record(): else var_["type"] ) - # keywords default to False, everything else to None + # keywords default to False, everything else to None. + # make sure strings are wrapped with quotation marks. var_["default"] = var.pop( "default", False if var_["type"] is bool else None ) + if isinstance(var_["default"], str): + default = var_["default"] + var_["default"] = f'"{default}"' # make substitutions from common variables # and remove backslashes from description @@ -1511,16 +1534,46 @@ def _qualify(var: dict) -> dict: var["children"] = {n: _qualify(c) for n, c in children.items()} return var + def _add_exg_vars(vars: dict) -> dict: + a = subcomponent[:3] + b = subcomponent[:3] + default = f"{a.upper()}6-{b.upper()}6" + return { + "exgtype": { + "name": "exgtype", + "type": "string", + "default": default, + "description": "The exchange type.", + "set_self": True, + }, + "exgmnamea": { + "name": "exgmnamea", + "type": "string", + "description": "The name of the first model in the exchange.", + "set_self": True, + }, + "exgmnameb": { + "name": "exgmnameb", + "type": "string", + "description": "The name of the second model in the exchange.", + "set_self": True, + }, + **vars, + } + def _variables(vars: dict) -> dict: """Get the class' member variables.""" - return { + vars_ = vars.copy() + vars_ = _add_exg_vars(vars_) if component == "exg" else vars_ + vars_ = { name: _qualify(_convert(var)) - for name, var in vars.items() + for name, var in vars_.items() # filter components of composites # since we've inflated the parent # types in the hierarchy already if not var.get("in_record", False) } + return vars_ def _dfn(vars: dict, meta: list) -> list: """ @@ -1545,6 +1598,8 @@ def _var_dfn(var: dict) -> List[str]: ] return { + "title": _title(component, subcomponent), + "parent": _parent(component, subcomponent), "component": component, "subcomponent": subcomponent, "variables": _variables(definition), @@ -1552,24 +1607,25 @@ def _var_dfn(var: dict) -> List[str]: } -def get_source_path(component, subcomponent): +def get_src_name(component: str, subcomponent: str): def _name(): - _case = (component, subcomponent) - if _case == ("sim", "nam"): - return "simulation" - elif _case == ("sim", "tdis"): - return "tdis" + if component == "sim": + if subcomponent == "nam": + return "simulation" + else: + return subcomponent elif component == "sln": return subcomponent + elif component == "exg": + return subcomponent return f"{component}{subcomponent}" - return SRCS_PATH / f"mf{_name()}.py" + return f"mf{_name()}.py" -def generate_component(dfn_path): +def generate_component(dfn_path: Path, out_path: Path, verbose: bool = False): comp, sub = dfn_path.stem.split("-") - py_path = get_source_path(comp, sub) - is_model = comp != "sim" and sub == "nam" + temp_type = TemplateType.from_pair(comp, sub) with open(DFNS_PATH / "common.dfn") as f: common_vars, _ = load_dfn(f) @@ -1580,44 +1636,53 @@ def generate_component(dfn_path): with open(dfn_path, "r") as f: vars, meta = load_dfn(f) - with open(py_path, "w") as f: + # write simulation or package class file + src_path = out_path / get_src_name(comp, sub) + with open(src_path, "w") as f: context = get_template_context( component=comp, subcomponent=sub, common_vars=common_vars, flopy_vars=flopy_vars, - variables=vars, + definition=vars, metadata=meta, ) - template = TEMPLATE_ENV.get_template( - f"{TemplateType.from_pair(comp, sub)}.jinja" - ) + template = TEMPLATE_ENV.get_template(f"{temp_type.value}.jinja") source = template.render(**context) + if verbose: + print(f"Generating {src_path} from {dfn_path}") f.write(source) - if is_model: - py_path = SRCS_PATH / f"mf{comp}.py" - with open(py_path, "w") as f: + if temp_type == TemplateType.Model: + # write model class file + src_path = out_path / get_src_name(comp, "") + with open(src_path, "w") as f: context = get_template_context( component=comp, subcomponent=sub, common_vars=common_vars, flopy_vars=flopy_vars, - variables=vars, + definition=vars, metadata=meta, ) - template = TEMPLATE_ENV.get_template( - f"{TemplateType.from_pair(comp, sub)}.jinja" - ) + template = TEMPLATE_ENV.get_template(f"{temp_type.value}.jinja") source = template.render(**context) + if verbose: + print(f"Generating {src_path} from {dfn_path}") f.write(source) -def generate_components(): - for dfn_path in DFNS_PATH.glob("*.dfn"): - generate_component(dfn_path) +def generate_components(out_path: Path, verbose: bool = False): + dfn_paths = [ + dfn + for dfn in DFNS_PATH.glob("*.dfn") + if dfn.stem not in ["common", "flopy"] + ] + for dfn_path in dfn_paths: + generate_component(dfn_path, out_path, verbose=verbose) + run_cmd("ruff", "format", out_path, verbose=verbose) if __name__ == "__main__": create_packages() - # generate_components + # generate_components(SRCS_PATH) diff --git a/pyproject.toml b/pyproject.toml index 0ab15b6415..436cbfcf27 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ classifiers = [ ] requires-python = ">=3.8" dependencies = [ - "Jinja2", + "Jinja2>=3.0", "numpy>=1.20.3", "matplotlib >=1.4.0", "pandas >=2.0.0", From a44890879999d21ba68be61014993eabf92dd234 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Fri, 27 Sep 2024 15:51:18 -0400 Subject: [PATCH 11/46] slinging dictionaries no longer --- .docs/code.rst | 4 +- .docs/md/generate_classes.md | 12 +- .github/workflows/commit.yml | 2 +- .github/workflows/examples.yml | 2 +- autotest/test_createpackages.py | 91 +- autotest/test_dfn.py | 15 - autotest/test_generate_classes.py | 2 +- docs/make_release.md | 2 +- docs/mf6_dev_guide.md | 2 +- flopy/mf6/{utils => }/createpackages.py | 1144 ++++++++++++----- flopy/mf6/{utils => }/generate_classes.py | 2 +- flopy/mf6/templates/attrs.jinja | 16 +- .../{package.jinja => context.jinja} | 9 +- flopy/mf6/templates/docstring.jinja | 6 +- flopy/mf6/templates/init.jinja | 67 +- flopy/mf6/templates/params.jinja | 12 +- flopy/mf6/utils/__init__.py | 2 - flopy/mf6/utils/dfn.py | 46 - 18 files changed, 960 insertions(+), 476 deletions(-) delete mode 100644 autotest/test_dfn.py rename flopy/mf6/{utils => }/createpackages.py (64%) rename flopy/mf6/{utils => }/generate_classes.py (99%) rename flopy/mf6/templates/{package.jinja => context.jinja} (60%) delete mode 100644 flopy/mf6/utils/dfn.py diff --git a/.docs/code.rst b/.docs/code.rst index 3bfeb040a6..f2d632154b 100644 --- a/.docs/code.rst +++ b/.docs/code.rst @@ -231,8 +231,8 @@ Contents: .. toctree:: :maxdepth: 4 - ./source/flopy.mf6.utils.createpackages.rst - ./source/flopy.mf6.utils.generate_classes.rst + ./source/flopy.mf6.createpackages.rst + ./source/flopy.mf6.generate_classes.rst Previous Versions of MODFLOW diff --git a/.docs/md/generate_classes.md b/.docs/md/generate_classes.md index 05c2300d40..c480b9b62d 100644 --- a/.docs/md/generate_classes.md +++ b/.docs/md/generate_classes.md @@ -13,7 +13,7 @@ MODFLOW 6 input continues to evolve as new models, packages, and options are dev The FloPy classes for MODFLOW 6 are largely generated by a utility which converts DFN files in a modflow6 repository on GitHub or on the local machine into Python source files in your local FloPy install. For instance (output much abbreviated): ```bash -$ python -m flopy.mf6.utils.generate_classes +$ python -m flopy.mf6.generate_classes @@ -55,7 +55,7 @@ Similar functionality is available within Python, e.g.: The `generate_classes()` function has several optional parameters. ```bash -$ python -m flopy.mf6.utils.generate_classes -h +$ python -m flopy.mf6.generate_classes -h usage: generate_classes.py [-h] [--owner OWNER] [--repo REPO] [--ref REF] [--dfnpath DFNPATH] [--no-backup] @@ -79,19 +79,19 @@ options: For example, use the develop branch instead: ```bash -$ python -m flopy.mf6.utils.generate_classes --ref develop +$ python -m flopy.mf6.generate_classes --ref develop ``` use a fork of modflow6: ```bash -$ python -m flopy.mf6.utils.generate_classes --owner your-username --ref your-branch +$ python -m flopy.mf6.generate_classes --owner your-username --ref your-branch ``` maybe your fork has a different name: ```bash -$ python -m flopy.mf6.utils.generate_classes --owner your-username --repo your-modflow6 --ref your-branch +$ python -m flopy.mf6.generate_classes --owner your-username --repo your-modflow6 --ref your-branch ``` local copy of the repo: ```bash -$ python -m flopy.mf6.utils.generate_classes --dfnpath ../your/dfn/path +$ python -m flopy.mf6.generate_classes --dfnpath ../your/dfn/path ``` Branch names, commit hashes, or tags may be provided to `ref`. diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml index 07e7b73bcc..e6c12ea268 100644 --- a/.github/workflows/commit.yml +++ b/.github/workflows/commit.yml @@ -175,7 +175,7 @@ jobs: subset: triangle - name: Update package classes - run: python -m flopy.mf6.utils.generate_classes --ref develop --no-backup + run: python -m flopy.mf6.generate_classes --ref develop --no-backup - name: Run tests working-directory: autotest diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 590a1578f3..fdf12b12f4 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -69,7 +69,7 @@ jobs: subset: triangle - name: Update FloPy packages - run: python -m flopy.mf6.utils.generate_classes --ref develop --no-backup + run: python -m flopy.mf6.generate_classes --ref develop --no-backup - name: Run example tests working-directory: autotest diff --git a/autotest/test_createpackages.py b/autotest/test_createpackages.py index 8fb191123e..7ec57b42c8 100644 --- a/autotest/test_createpackages.py +++ b/autotest/test_createpackages.py @@ -2,75 +2,84 @@ from modflow_devtools.misc import run_cmd from autotest.conftest import get_project_root_path -from flopy.mf6.utils.createpackages import ( +from flopy.mf6.createpackages import ( TEMPLATE_ENV, - TemplateType, - generate_components, - get_src_name, + ContextType, + DefinitionName, + generate_targets, get_template_context, + load_dfn, ) -from flopy.mf6.utils.dfn import load_dfn +from flopy.mf6.mfpackage import MFPackage PROJ_ROOT = get_project_root_path() -DFNS_PATH = PROJ_ROOT / "flopy" / "mf6" / "data" / "dfn" +DFN_PATH = PROJ_ROOT / "flopy" / "mf6" / "data" / "dfn" DFNS = [ dfn - for dfn in DFNS_PATH.glob("*.dfn") + for dfn in DFN_PATH.glob("*.dfn") if dfn.stem not in ["common", "flopy"] ] +@pytest.mark.parametrize("dfn", DFNS) +def test_load_dfn(dfn): + dfn_path = DFN_PATH / dfn + with open(dfn_path, "r") as f: + definition = load_dfn(f) + + # only test packages for which we know the -# expected consolidated number of variables +# expected number of consolidated variables @pytest.mark.parametrize( - "dfn, n_flat, n_nested", [("gwf-ic", 2, 2), ("prt-prp", 40, 18)] + "dfn, n_flat, n_params", [("gwf-ic", 2, 6), ("prt-prp", 40, 22)] ) -def test_get_template_context(dfn, n_flat, n_nested): - component, subcomponent = dfn.split("-") - - with open(DFNS_PATH / "common.dfn") as f: - common_vars, _ = load_dfn(f) +def test_get_template_context(dfn, n_flat, n_params): + dfn_name = DefinitionName(*dfn.split("-")) - with open(DFNS_PATH / "flopy.dfn") as f: - flopy_vars, _ = load_dfn(f) + with open(DFN_PATH / "common.dfn") as f: + common, _ = load_dfn(f) - with open(DFNS_PATH / f"{dfn}.dfn") as f: - variables, metadata = load_dfn(f) + with open(DFN_PATH / f"{dfn}.dfn") as f: + definition = load_dfn(f) context = get_template_context( - component, subcomponent, common_vars, flopy_vars, variables, metadata + dfn_name, + MFPackage, + common, + definition, ) - assert context["component"] == component - assert context["subcomponent"] == subcomponent - assert len(context["variables"]) == n_nested + assert context["name"] == dfn_name + assert len(context["parameters"]) == n_params assert len(context["dfn"]) == n_flat + 1 # +1 for metadata @pytest.mark.parametrize("dfn", [dfn.stem for dfn in DFNS]) def test_render_template(dfn, function_tmpdir): - component, subcomponent = dfn.split("-") - context_name = f"{component}{subcomponent}" - template_type = TemplateType.from_pair(component, subcomponent).value - template = TEMPLATE_ENV.get_template(f"{template_type}.jinja") - - with open(DFNS_PATH / "common.dfn") as f: - common_vars, _ = load_dfn(f) - - with open(DFNS_PATH / "flopy.dfn") as f: - flopy_vars, _ = load_dfn(f) - - with open(DFNS_PATH / f"{dfn}.dfn", "r") as f: - variables, metadata = load_dfn(f) - - context = get_template_context( - component, subcomponent, common_vars, flopy_vars, variables, metadata - ) + dfn_name = DefinitionName(*dfn.split("-")) + context_type = ContextType.from_dfn_name(dfn_name) + template = TEMPLATE_ENV.get_template("context.jinja") + + with open(DFN_PATH / "common.dfn") as f: + common, _ = load_dfn(f) + + with open(DFN_PATH / f"{dfn}.dfn", "r") as f: + definition = load_dfn(f) + + context = { + "context": context_type.value, + **get_template_context( + dfn_name, + context_type.base, + common, + definition, + ), + } source = template.render(**context) - source_path = function_tmpdir / get_src_name(component, subcomponent) + source_path = function_tmpdir / dfn_name.target with open(source_path, "w") as f: f.write(source) run_cmd("ruff", "format", source_path, verbose=True) def test_generate_components(function_tmpdir): - generate_components(function_tmpdir, verbose=True) + generate_targets(DFN_PATH, function_tmpdir, verbose=True) diff --git a/autotest/test_dfn.py b/autotest/test_dfn.py deleted file mode 100644 index 0151341a9d..0000000000 --- a/autotest/test_dfn.py +++ /dev/null @@ -1,15 +0,0 @@ -import pytest - -from autotest.conftest import get_project_root_path -from flopy.mf6.utils.dfn import load_dfn - -PROJ_ROOT = get_project_root_path() -DFNS_PATH = PROJ_ROOT / "flopy" / "mf6" / "data" / "dfn" -DFNS = [dfn.name for dfn in DFNS_PATH.glob("*.dfn")] - - -@pytest.mark.parametrize("dfn", DFNS) -def test_load_dfn(dfn): - dfn_path = DFNS_PATH / dfn - with open(dfn_path, "r") as f: - vars, meta = load_dfn(f) diff --git a/autotest/test_generate_classes.py b/autotest/test_generate_classes.py index db812aa40b..44cac88ca1 100644 --- a/autotest/test_generate_classes.py +++ b/autotest/test_generate_classes.py @@ -122,7 +122,7 @@ def test_generate_classes_from_github_refs( out, err, ret = run_cmd( str(python), "-m", - "flopy.mf6.utils.generate_classes", + "flopy.mf6.generate_classes", "--owner", owner, "--repo", diff --git a/docs/make_release.md b/docs/make_release.md index d65c7619f5..568ebe763c 100644 --- a/docs/make_release.md +++ b/docs/make_release.md @@ -96,7 +96,7 @@ As described above, making a release manually involves the following steps: - Run `python scripts/update_version.py -v ` to update the version number stored in `version.txt` and `flopy/version.py`. For an approved release use the `--approve` flag. -- Update MODFLOW 6 dfn files in the repository and MODFLOW 6 package classes by running `python -m flopy.mf6.utils.generate_classes --ref master --no-backup` +- Update MODFLOW 6 dfn files in the repository and MODFLOW 6 package classes by running `python -m flopy.mf6.generate_classes --ref master --no-backup` - Run `ruff check .` and `ruff format .` from the project root. diff --git a/docs/mf6_dev_guide.md b/docs/mf6_dev_guide.md index 8f95ee9bbb..60aff50bc4 100644 --- a/docs/mf6_dev_guide.md +++ b/docs/mf6_dev_guide.md @@ -6,7 +6,7 @@ This file provides an overview of how FloPy for MODFLOW 6 (FPMF6) works under th Package Meta-Data and Package Files ----------------------------------------------- -FPMF6 uses meta-data files located in flopy/mf6/data/dfn to define the model and package types supported by MODFLOW 6. When additional model and package types are added to MODFLOW 6, additional meta-data files can be added to this folder and flopy/mf6/utils/createpackages.py can be run to add new packages to the FloPy library. createpackages.py uses flopy/mf6/data/mfstructure.py to read meta-data files (*.dfn) and use that meta-data to create the package files found in flopy/mf6/modflow (do not directly modify any of the files in this folder, they are all automatically generated). The automatically generated package files contain an interface for accessing package data and data documentation generated from the meta-data files. Additionally, meta-data describing package data types and shapes is stored in the dfn attribute. flopy/mf6/data/mfstructure.py can load structure information using the dfn attribute (instead of loading it from the meta-data files). This allows for flopy to be installed without the dfn files. +FPMF6 uses meta-data files located in flopy/mf6/data/dfn to define the model and package types supported by MODFLOW 6. When additional model and package types are added to MODFLOW 6, additional meta-data files can be added to this folder and flopy/mf6/createpackages.py can be run to add new packages to the FloPy library. createpackages.py uses flopy/mf6/data/mfstructure.py to read meta-data files (*.dfn) and use that meta-data to create the package files found in flopy/mf6/modflow (do not directly modify any of the files in this folder, they are all automatically generated). The automatically generated package files contain an interface for accessing package data and data documentation generated from the meta-data files. Additionally, meta-data describing package data types and shapes is stored in the dfn attribute. flopy/mf6/data/mfstructure.py can load structure information using the dfn attribute (instead of loading it from the meta-data files). This allows for flopy to be installed without the dfn files. All meta-data can be accessed from the flopy.mf6.data.mfstructure.MFStructure class. This is a singleton class, meaning only one instance of this class can be created. The class contains a sim_struct attribute (which is a flopy.mf6.data.mfstructure.MFSimulationStructure object) which contains all of the meta-data for all package files. Meta-data is stored in a structured format. MFSimulationStructure contains MFModelStructure and MFInputFileStructure objects, which contain the meta-data for each model type and each "simulation-level" package (tdis, ims, ...). MFModelStructure contains model specific meta-data and a MFInputFileStructure object for each package in that model. MFInputFileStructure contains package specific meta-data and a MFBlockStructure object for each block contained in the package file. MFBlockStructure contains block specific meta-data and a MFDataStructure object for each data structure defined in the block, and MFDataStructure contains data structure specific meta-data and a MFDataItemStructure object for each data item contained in the data structure. Data structures define the structure of data that is naturally grouped together, for example, the data in a numpy recarray. Data item structures define the structure of specific pieces of data, for example, a single column of a numpy recarray. The meta-data defined in these classes provides all the information FloPy needs to read and write MODFLOW 6 package and name files, create the Flopy interface, and check the data for various constraints. diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/createpackages.py similarity index 64% rename from flopy/mf6/utils/createpackages.py rename to flopy/mf6/createpackages.py index 7a43383f48..cd8312c1e5 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/createpackages.py @@ -85,30 +85,41 @@ import datetime import os import textwrap +from abc import ABC, abstractmethod +from dataclasses import asdict, dataclass, replace from enum import Enum from keyword import kwlist +from os import PathLike from pathlib import Path from typing import ( + Any, Dict, ForwardRef, Iterable, List, Literal, + NamedTuple, Optional, Tuple, + TypedDict, Union, get_args, get_origin, ) +from warnings import warn import numpy as np from jinja2 import Environment, PackageLoader from modflow_devtools.misc import run_cmd -from numpy.typing import NDArray +from numpy.typing import ArrayLike, NDArray + +from flopy.mf6 import MFSimulation # keep below as absolute imports from flopy.mf6.data import mfdatautil, mfstructure -from flopy.mf6.utils.dfn import Definition, load_dfn +from flopy.mf6.mfmodel import MFModel +from flopy.mf6.mfpackage import MFPackage +from flopy.mf6.mfsimbase import MFSimulationBase from flopy.utils import datautil @@ -858,7 +869,7 @@ def create_packages(): local_datetime = datetime.datetime.now(datetime.timezone.utc) comment_string = ( "# DO NOT MODIFY THIS FILE DIRECTLY. THIS FILE " - "MUST BE CREATED BY\n# mf6/utils/createpackages.py\n" + "MUST BE CREATED BY\n# mf6/createpackages.py\n" "# FILE created on {} UTC".format( local_datetime.strftime("%B %d, %Y %H:%M:%S") ) @@ -1153,8 +1164,11 @@ def create_packages(): init_file.close() -DFNS_PATH = Path(__file__).parents[1] / "data" / "dfn" -SRCS_PATH = Path(__file__).parents[1] / "modflow" +# ------------------------------------------ + + +DFN_PATH = Path(__file__).parent / "data" / "dfn" +TGT_PATH = Path(__file__).parent / "modflow" SCALAR_TYPES = { "keyword": bool, "integer": int, @@ -1170,22 +1184,147 @@ def create_packages(): TEMPLATE_ENV = Environment(loader=PackageLoader("flopy", "mf6/templates/")) -class TemplateType(Enum): +Scalar = Union[bool, int, float, str] +Variable = Dict[str, Scalar] +Variables = Dict[str, Variable] +Metadata = List[str] +Definition = Tuple[Variables, Metadata] +Definitions = Dict[str, Definition] + + +def load_dfn(f) -> Definition: + """ + Load an input definition file. Returns a tuple containing + a dictionary variables and a list of metadata attributes. + """ + meta = list() + vars = dict() + var = dict() + + for line in f: + # remove whitespace/etc from the line + line = line.strip() + + # record flopy metadata attributes but + # skip all other comment lines + if line.startswith("#"): + _, sep, tail = line.partition("flopy") + if sep == "flopy": + meta.append(tail.strip()) + continue + + # if we hit a newline and the parameter dict + # is nonempty, we've reached the end of its + # block of attributes + if not any(line): + if any(var): + vars[var["name"]] = var + var = dict() + continue + + # split the attribute's key and value and + # store it in the parameter dictionary + key, _, value = line.partition(" ") + var[key] = value + + # add the final parameter + if any(var): + vars[var["name"]] = var + + return vars, meta + + +class DefinitionName(NamedTuple): + l: str + r: str + + @property + def title(self) -> str: + l, r = self + if l == "sln": + return r + elif l == "exg": + return r + return f"{l}{r}" + + @property + def parent(self) -> type: + l, r = self + if l in ["exg", "sln"]: + return MFSimulation + if l in ["utl"]: + return MFPackage + return MFModel + + @property + def parent_kind(self) -> str: + return self.parent.__name__.lower().replace("mf", "") + + @property + def description(self) -> str: + l, r = self + title = self.title.title() + parent = self.parent + parent_kind = self.parent_kind + context = ContextType.from_dfn_name(self) + if context == ContextType.Package: + return ( + f"Modflow{title} defines a {r} package " + f"within a {l} {parent_kind}." + ) + elif context == ContextType.Model: + return f"Modflow{title} defines a {l.upper()} model." + elif context == ContextType.Simulation: + return """ + MFSimulation is used to load, build, and/or save a MODFLOW 6 simulation. + A MFSimulation object must be created before creating any of the MODFLOW 6 + model objects.""" + + @property + def target(self) -> str: + l, r = self + + def _name(): + if l == "sim": + if r == "nam": + return "simulation" + else: + return r + elif l == "sln": + return r + elif l == "exg": + return r + return f"{l}{r}" + + return f"mf{_name()}.py" + + +class ContextType(Enum): Model = "model" Package = "package" Simulation = "simulation" + @property + def base(self): + if self == ContextType.Model: + return MFModel + elif self == ContextType.Package: + return MFPackage + elif self == ContextType.Simulation: + return MFSimulationBase + @classmethod - def from_pair(cls, component, subcomponent) -> "TemplateType": - if component == "sim" and subcomponent == "nam": - return TemplateType.Simulation - elif subcomponent == "nam": - return TemplateType.Model + def from_dfn_name(cls, dfn_name: DefinitionName) -> "ContextType": + l, r = dfn_name + if l == "sim" and r == "nam": + return ContextType.Simulation + elif r == "nam": + return ContextType.Model else: - return TemplateType.Package + return ContextType.Package -def fullname(t: type) -> str: +def _try_qualify(t) -> str: """Convert a type to a name suitable for templating.""" origin = get_origin(t) args = get_args(t) @@ -1194,16 +1333,20 @@ def fullname(t: type) -> str: return f"{Literal.__name__}[{', '.join(args)}]" elif origin is Union: if len(args) == 2 and args[1] is type(None): - return f"{Optional.__name__}[{fullname(args[0])}]" - return f"{Union.__name__}[{', '.join([fullname(a) for a in args])}]" + return f"{Optional.__name__}[{_try_qualify(args[0])}]" + return ( + f"{Union.__name__}[{', '.join([_try_qualify(a) for a in args])}]" + ) elif origin is tuple: - return f"{Tuple.__name__}[{', '.join([fullname(a) for a in args])}]" + return ( + f"{Tuple.__name__}[{', '.join([_try_qualify(a) for a in args])}]" + ) elif origin is collections.abc.Iterable: - return f"{Iterable.__name__}[{', '.join([fullname(a) for a in args])}]" + return f"{Iterable.__name__}[{', '.join([_try_qualify(a) for a in args])}]" elif origin is list: - return f"{List.__name__}[{', '.join([fullname(a) for a in args])}]" + return f"{List.__name__}[{', '.join([_try_qualify(a) for a in args])}]" elif origin is np.ndarray: - return f"NDArray[np.{fullname(args[1].__args__[0])}]" + return f"NDArray[np.{_try_qualify(args[1].__args__[0])}]" elif origin is np.dtype: return str(t) elif isinstance(t, ForwardRef): @@ -1212,48 +1355,204 @@ def fullname(t: type) -> str: return "..." elif isinstance(t, type): return t.__qualname__ - else: - return str(t) + return t + + +def _try_enum_value(v: Any) -> Any: + return v.value if isinstance(v, Enum) else v + + +def _try_de_underscore(k: str) -> str: + return k[1:] if k.startswith("_") else k + + +def _render(d: dict) -> dict: + """ + Recursively transform a dictionary for rendering in a template. + """ + + def _render_key(k): + return _try_de_underscore(k) + + def _render_val(v): + return ( + _render(v) + if isinstance(v, dict) + else _try_qualify(_try_enum_value(v)) + ) + + # drop nones, except for default + _d = { + _render_key(k): _render_val(v) + for k, v in d.items() + if (k == "default" or v is not None) + } + + # wrap default in quotation marks if it's a string + default = _d.get("default", None) + if default is not None and isinstance(default, str): + _d["default"] = f'"{default}"' + + return _d + + +class Renderable: + def render(self) -> dict: + """ + Recursively transform the dataclass instance for rendering. + """ + return _render(asdict(self)) + + +@dataclass(frozen=True) +class Subpackage(Renderable): + parent: type + abbreviation: str + data_name: str + param_name: str + record_name: str + + +class ParameterKind(Enum): + Array = "array" + Scalar = "scalar" + Record = "record" + Union = "union" + List = "list" + + @classmethod + def from_type(cls, t: type) -> Optional["ParameterKind"]: + origin = get_origin(t) + try: + if t is np.ndarray or origin is NDArray or origin is ArrayLike: + return ParameterKind.Array + elif origin is collections.abc.Iterable or origin is list: + return ParameterKind.List + elif origin is tuple: + return ParameterKind.Record + elif origin is Union: + return ParameterKind.Union + elif issubclass(t, Scalar): + return ParameterKind.Scalar + return None + except: + pass + + +@dataclass +class Parameter(Renderable): + name: str + _type: type + description: Optional[str] + default: Optional[Any] + children: Optional[Dict[str, "Parameter"]] + subpackage: Optional[Subpackage] + kind: Optional[ParameterKind] + is_choice: bool = False + set_self: bool = False + + def __init__( + self, + name: str, + _type: type, + description: Optional[str] = None, + default: Optional[Any] = None, + parent: Optional["Parameter"] = None, + children: Optional[Dict[str, "Parameter"]] = None, + subpackage: Optional[Subpackage] = None, + kind: Optional[ParameterKind] = None, + is_choice: bool = False, + set_self: bool = False, + ): + self.name = name + self._type = _type + self.description = description + self.default = default + self.parent = parent + self.children = children + self.subpackage = subpackage + # note the general kind of the parameter, + # i.e. the general type theoretic concept. + # this is ofc derivable on demand, but Jinja + # doesn't allow arbitrary expressions, and it + # doesn't seem to have `subclass`-ish filters. + self.kind = kind or ParameterKind.from_type(_type) + self.is_choice = is_choice + self.set_self = set_self + + +Parameters = Dict[str, Parameter] + + +def _try_get_subpkg(metadata: Metadata) -> Optional[Subpackage]: + lines = { + "subpkg": next( + iter(m for m in metadata if m.startswith("subpac")), None + ), + "parent": next( + iter(m for m in metadata if m.startswith("parent")), None + ), + } + + def _subpkg(): + line = lines["subpkg"] + _, record_name, abbreviation, param_name, data_name = line.split() + return { + "abbreviation": abbreviation, + "data_name": data_name, + "param_name": param_name, + "record_name": record_name, + } + + def _parent(): + line = lines["parent"] + _, _, param_type = line.split() + param_type = param_type.lower() + if "simulation" in param_type: + return MFSimulation + elif "model" in param_type: + return MFModel + return MFPackage + + return ( + Subpackage(parent=_parent(), **_subpkg()) + if all(v for v in lines.values()) + else None + ) def get_template_context( - component: str, - subcomponent: str, - common_vars: Definition, - flopy_vars: Definition, + name: DefinitionName, + base: Union[MFSimulationBase, MFModel, MFPackage], + common: Variables, definition: Definition, - metadata: List[str], + references: Optional[Definitions] = None, ) -> dict: """ Convert an input definition to a template rendering context. - TODO: pull out a class for the input definition, and expose - this as an instance method? + Notes + ----- + Each input definition corresponds to a generated file/class. + + A map of other definitions may be provided, in which case a + parameter in this context may act as kind of "foreign key", + identifying another context (e.g. a subpackage) which this + context "owns". This allows conditionally initializing any + subpackages in `__init__` for input contexts that use them. """ - def _title(comp, sub): - if comp == "sln": - return sub.title() - elif component == "exg": - return sub.title() - return f"{comp.title()}{sub}" - - def _parent(comp, sub): - if comp in ["exg", "sln"]: - return {"name": "simulation", "type": "MFSimulation"} - if comp in ["utl"]: - return {"name": "package", "type": "MFPackage"} - return {"name": "model", "type": "MFModel"} - - def _convert(var: dict, wrap: bool = False) -> dict: + variables, metadata = definition + + def _convert(variable: Variable, wrap: bool = False) -> Parameter: """ Transform a variable from its original representation in - an input definition to a form suitable for type hints and - and docstrings. + an input definition to a parameter for inclusion in type + hints, docstrings, and the `__init__` method's signature. - This involves expanding nested type hierarchies, converting - input types to equivalent Python primitives and composites, - and various other shaping. + This involves expanding nested input hierarchies, mapping + types to roughly equivalent Python primitives/composites, + and other shaping. Notes ----- @@ -1264,33 +1563,39 @@ def _convert(var: dict, wrap: bool = False) -> dict: keywords represented as string literals. This is useful for unions, to distinguish between choices having the same type. """ - var_ = { - **var, - # some flags the template uses for formatting. - # these are ofc derivable in Python but Jinja - # doesn't allow arbitrary expressions, and it - # doesn't seem to have `subclass`-ish filters. - # (we convert the variable type to string too - # before returning, for the same reason.) - "is_array": False, - "is_list": False, - "is_record": False, - "is_union": False, - "is_iterable": False, - "is_choice": False, - } - name_ = var["name"] - type_ = var["type"] - shape = var.get("shape", None) + + _name = variable["name"] + _type = variable.get("type", "unknown") + shape = variable.get("shape", None) shape = None if shape == "" else shape + optional = variable.get("optional", True) + in_record = variable.get("in_record", False) + tagged = variable.get("tagged, False") + description = variable.get("description", "") + children = None - # utilities for generating records - # as named tuples. + def _description(description: str) -> str: + """ + Make substitutions from common variables, + remove backslashes from the description. + TODO: insert citations. + """ + description = description.replace("\\", "") + _, replace, tail = description.strip().partition("REPLACE") + if replace: + key, _, replacements = tail.strip().partition(" ") + replacements = eval(replacements) + val = common.get(key, None).get("description", "") + if val is None: + raise ValueError(f"Common variable not found: {key}") + if any(replacements): + return val.replace("{#1}", replacements["{#1}"]) + return val + return description - def _get_record_fields(name: str) -> dict: + def _fields(record_name: str) -> Parameters: """ - Call `_map_var` recursively on each field - of the record variable with the given name. + Load a record's fields and recursively convert them. Notes ----- @@ -1299,17 +1604,17 @@ def _get_record_fields(name: str) -> dict: and 'filein'/'fileout', which are details of the mf6io format, not of python/flopy. """ - record = definition[name] - names = record["type"].split()[1:] - fields = { - n: {**_convert(field), "optional": field.get("optional", True)} - for n, field in definition.items() - if n in names + record = variables[record_name] + field_names = record["type"].split()[1:] + fields: Dict[str, Parameter] = { + n: _convert(field) + for n, field in variables.items() + if n in field_names } field_names = list(fields.keys()) # if the record represents a file... - if "file" in name: + if "file" in record_name: # remove filein/fileout for term in ["filein", "fileout"]: if term in field_names: @@ -1322,9 +1627,9 @@ def _get_record_fields(name: str) -> dict: # set the type n = list(fields.keys())[0] - path = fields[n] - path["type"] = os.PathLike - fields[n] = path + path_field = fields[n] + path_field._type = os.PathLike + fields[n] = path_field # if tagged, remove the leading keyword elif record.get("tagged", False): @@ -1334,15 +1639,23 @@ def _get_record_fields(name: str) -> dict: return fields + # now, proceed through the possible input types + # from top (composites) to bottom (primitives): + # - list + # - union + # - record + # - array + # - scalar + # list input can have records or unions as rows. # lists which have a consistent record type are # regular, inconsistent record types irregular. - if type_.startswith("recarray"): + if _type.startswith("recarray"): # make sure columns are defined - names = type_.split()[1:] + names = _type.split()[1:] n_names = len(names) if n_names < 1: - raise ValueError(f"Missing recarray definition: {type_}") + raise ValueError(f"Missing recarray definition: {_type}") # regular tabular/columnar data (1 record type) can be # defined with a nested record (i.e. explicit) or with @@ -1350,234 +1663,365 @@ def _get_record_fields(name: str) -> dict: # data for unions/keystrings necessarily comes nested. def _is_explicit_record(): - return len(names) == 1 and definition[names[0]][ + return len(names) == 1 and variables[names[0]][ "type" ].startswith("record") def _is_implicit_record(): types = [ - fullname(v["type"]) - for n, v in definition.items() + _try_qualify(v["type"]) + for n, v in variables.items() if n in names ] scalar_types = list(SCALAR_TYPES.keys()) return all(t in scalar_types for t in types) if _is_explicit_record(): - name = names[0] - record_type = _convert(definition[name]) - var_["type"] = List[record_type["type"]] - var_["children"] = {name: record_type} - var_["is_list"] = True + record_name = names[0] + record_type = _convert(variables[record_name]) + children = {record_name: record_type} + type_ = Iterable[record_type._type] elif _is_implicit_record(): # record implicitly defined, make it on the fly - name = name_ - fields = _get_record_fields(name) + record_name = _name + record_fields = _fields(record_name) record_type = Tuple[ - tuple([f["type"] for f in fields.values()]) + tuple([f._type for f in record_fields.values()]) ] - record = { - "name": name, - "type": record_type, - "children": fields, - "is_array": False, - "is_record": True, - "is_union": False, - "is_list": False, - "is_iterable": False, - "is_choice": False, - } - var_["type"] = List[record_type] - var_["children"] = {name: record} - var_["is_list"] = True + record = Parameter( + name=record_name, + _type=record_type, + children=record_fields, + ) + children = {record_name: record} + type_ = Iterable[record_type] else: # irregular recarray, rows can be any of several types - children = {n: _convert(definition[n]) for n in names} - var_["type"] = List[ - Union[tuple([c["type"] for c in children.values()])] + children = {n: _convert(variables[n]) for n in names} + type_ = Iterable[ + Union[tuple([c._type for c in children.values()])] ] - var_["children"] = children - var_["is_list"] = True - - # now the basic composite types... - # union (product) type, children are choices of records - elif type_.startswith("keystring"): - names = type_.split()[1:] - children = {n: _convert(definition[n], wrap=True) for n in names} - var_["type"] = Union[tuple([c["type"] for c in children.values()])] - var_["children"] = children - var_["is_union"] = True + + # basic composite types... + # union (product), children are record choices + elif _type.startswith("keystring"): + names = _type.split()[1:] + children = {n: _convert(variables[n], wrap=True) for n in names} + type_ = Union[tuple([c._type for c in children.values()])] # record (sum) type, children are fields - elif type_.startswith("record"): - name = name_ - fields = _get_record_fields(name) - if len(fields) > 1: + elif _type.startswith("record"): + children = _fields(_name) + if len(children) > 1: record_type = Tuple[ - tuple([c["type"] for c in fields.values()]) + tuple([c._type for c in children.values()]) ] - elif len(fields) == 1: - t = list(fields.values())[0]["type"] + elif len(children) == 1: + t = list(children.values())[0]._type # make sure we don't double-wrap tuples record_type = t if get_origin(t) is tuple else Tuple[(t,)] # TODO: if record has 1 field, accept value directly? - var_["type"] = record_type - var_["children"] = fields - var_["is_record"] = True + type_ = record_type - # are we wrapping a choice in a union? - # if so, use a literal for the leading - # keyword like tuple (Literal[...], T) + # are we wrapping a record which is a + # choice within a union? if so, use a + # literal for the keyword as tag e.g. + # `Tuple[Literal[...], T]` elif wrap: - name = name_ - field = _convert(var) - fields = {name: field} + field_name = _name + field = _convert(variable) field_type = ( - Literal[name] if field["type"] is bool else field["type"] + Literal[field_name] if field._type is bool else field._type ) record_type = ( - Tuple[Literal[name]] - if field["type"] is bool - else Tuple[Literal[name], field["type"]] + Tuple[Literal[field_name]] + if field._type is bool + else Tuple[Literal[field_name], field._type] ) - fields[name] = {**field, "type": field_type} - var_["type"] = record_type - var_["children"] = fields - var_["is_record"] = True - var_["is_choice"] = True - - # at this point, if it has a shape, it's an array. - # but if it's in a record use a variadic tuple. + children = { + field_name: replace(field, _type=field_type, is_choice=True) + } + type_ = record_type + + # at this point, if it has a shape, it's an array.. + # but if it's in a record make it a variadic tuple, + # and if its item type is a string use an iterable. elif shape is not None: scalars = list(SCALAR_TYPES.keys()) - if var.get("in_record", False): - if type_ not in scalars: - raise TypeError(f"Unsupported repeating type: {type_}") - var_["type"] = Tuple[SCALAR_TYPES[type_], ...] - elif type_ in scalars: - var_["type"] = Iterable[SCALAR_TYPES[type_]] - var_["is_iterable"] = True + if in_record: + if _type not in scalars: + raise TypeError(f"Unsupported repeating type: {_type}") + type_ = Tuple[SCALAR_TYPES[_type], ...] + elif _type in scalars and SCALAR_TYPES[_type] is str: + type_ = Iterable[SCALAR_TYPES[_type]] else: - if type_ not in NP_SCALAR_TYPES.keys(): - raise TypeError(f"Unsupported array type: {type_}") - var_["type"] = NDArray[NP_SCALAR_TYPES[type_]] - var_["is_array"] = True + if _type not in NP_SCALAR_TYPES.keys(): + raise TypeError(f"Unsupported array type: {_type}") + type_ = NDArray[NP_SCALAR_TYPES[_type]] # finally a bog standard scalar else: - # if it's a keyword tag for another - # variable, make it a string literal - tag = type_ == "keyword" and (wrap or var.get("tagged", False)) - var_["type"] = Literal[name_] if tag else SCALAR_TYPES[type_] + # if it's a keyword, there are two cases we want to convert it to + # a string literal: if it's 1) tagging another variable, or 2) it + # is being wrapped into a record to represent a choice in a union + tag = _type == "keyword" and (tagged or wrap) + type_ = Literal[_name] if tag else SCALAR_TYPES.get(_type, _type) # make optional if needed - if var_.get("optional", True): - var_["type"] = ( - Optional[var_["type"]] - if ( - var_["type"] is not bool - and var_.get("optional", True) - and not var_.get("in_record", False) - and not wrap - ) - else var_["type"] + if optional: + type_ = ( + Optional[type_] + if (type_ is not bool and not in_record and not wrap) + else type_ ) - # keywords default to False, everything else to None. - # make sure strings are wrapped with quotation marks. - var_["default"] = var.pop( - "default", False if var_["type"] is bool else None - ) - if isinstance(var_["default"], str): - default = var_["default"] - var_["default"] = f'"{default}"' + # keywords default to False, everything else to None + default = variable.get("default", False if type_ is bool else None) + + return Parameter( + # if name is a reserved keyword, add a trailing underscore to it + name=f"{_name}_" if _name in kwlist else _name, + # MF6IO type is mapped to a (lenient) Python equivalent + _type=type_, + # default value + default=default, + # do substitutions, expand citations, apply formatting, etc + description=_description(description), + # child parameters + children=children, + # does variable reference another context (e.g. subpkg)? + subpackage=references.get(_name, None) if references else None, + ) - # make substitutions from common variables - # and remove backslashes from description - def _map_descr(description: str) -> str: - description = description.replace("\\", "") - _, replace, tail = description.strip().partition("REPLACE") - if replace: - key, _, replacements = tail.strip().partition(" ") - replacements = eval(replacements) - val = common_vars.get(key, None).get("description", "") - if val is None: - raise ValueError(f"Common variable not found: {key}") - if any(replacements): - return val.replace("{#1}", replacements["{#1}"]) - return val - return description + def _add_exg_params(params: Parameters) -> Parameters: + a = name.r[:3] + b = name.r[:3] + default = f"{a.upper()}6-{b.upper()}6" + return { + "exgtype": Parameter( + **{ + "name": "exgtype", + "_type": str, + "default": default, + "description": "The exchange type.", + "set_self": True, + } + ), + "exgmnamea": Parameter( + **{ + "name": "exgmnamea", + "_type": str, + "description": "The name of the first model in the exchange.", + "set_self": True, + } + ), + "exgmnameb": Parameter( + **{ + "name": "exgmnameb", + "_type": str, + "description": "The name of the second model in the exchange.", + "set_self": True, + } + ), + **params, + } - var_["description"] = _map_descr(var_.get("description", "")) + def _add_pkg_params(params: Parameters) -> Parameters: + parent_type = name.parent + parent_name = parent_type.__name__.lower().replace("mf", "") + is_subpkg = name[0] in ["utl"] + out = { + parent_name: Parameter( + **{ + "name": parent_name, + "_type": parent_type, + "description": ( + f"{parent_name.title()} that this package is part of. " + f"Package is automatically added to the {parent_name} " + "when it is initialized." + ), + } + ), + "loading_package": Parameter( + **{ + "name": "loading_package", + "_type": bool, + "description": ( + "Do not set this variable. It is intended for debugging " + "and internal processing purposes only." + ), + } + ), + **params, + "filename": Parameter( + **{ + "name": "filename", + "_type": str, + "description": "File name for this package.", + } + ), + "pname": Parameter( + **{ + "name": "pname", + "_type": str, + "description": "Package name for this package.", + } + ), + } + if is_subpkg: + out["parent_file"] = Parameter( + **{ + "name": "parent_file", + "_type": PathLike, + "description": ( + "Parent package file that references this package. Only needed " + "for utility packages (mfutl*). For example, mfutllaktab package " + "must have a mfgwflak package parent_file." + ), + } + ) + return out - # if name is a reserved keyword, add a trailing underscore to it - var_["name"] = ( - f"{var_['name']}_" if var_["name"] in kwlist else var_["name"] - ) + def _add_mdl_params(params: Parameters) -> Parameters: + return { + "simulation": Parameter( + **{ + "name": "simulation", + "_type": MFSimulation, + "description": ( + "Simulation that this model is part of. " + "Model is automatically added to the simulation " + "when it is initialized." + ), + } + ), + "modelname": Parameter( + **{ + "name": "modelname", + "_type": str, + "description": "The name of the model.", + } + ), + "model_nam_file": Parameter( + **{ + "name": "model_nam_file", + "_type": PathLike, + "description": ( + "The relative path to the model name file from model working folder." + ), + } + ), + "version": Parameter( + **{ + "name": "version", + "_type": str, + "description": "The version of modflow", + } + ), + "exe_name": Parameter( + **{ + "name": "exe_name", + "_type": str, + "description": "The executable name.", + } + ), + "model_ws": Parameter( + **{ + "name": "model_ws", + "_type": PathLike, + "description": "The model working folder path.", + } + ), + **params, + } - return var_ + def _add_sim_params(params: Parameters) -> Parameters: + sim_params = { + "sim_name": Parameter( + name="sim_name", + _type=str, + default="sim", + description="Name of the simulation.", + ), + "version": Parameter( + name="version", + _type=str, + default="mf6", + ), + "exe_name": Parameter( + name="exe_name", + _type=PathLike, + default="mf6", + ), + "sim_ws": Parameter( + name="sim_ws", + _type=PathLike, + default=os.curdir, + ), + "verbosity_level": Parameter( + name="verbosity_level", + _type=int, + default=1, + ), + "write_headers": Parameter( + name="write_headers", + _type=bool, + default=True, + ), + "use_pandas": Parameter( + name="use_pandas", + _type=bool, + default=True, + ), + "lazy_io": Parameter( + name="lazy_io", + _type=bool, + default=False, + ), + } + return { + **sim_params, + **params, + } - def _qualify(var: dict) -> dict: + def _parameters(definition: Definition) -> Parameters: """ - Recursively convert the variable's type to a fully qualified string. + Convert the input variables to parameters for an input + context class. Context-specific parameters may also be + added depending. Notes ----- - Separate function operating on the entire variable (rather than the - type alone) because we want to pass typed definitions around until - conversion just before templating. + Not all variables become parameters; nested variables + will become components of composite parameters, e.g., + record fields, keystring (union) choices, list items. """ - var["type"] = fullname(var["type"]) - children = var.get("children", dict()) - if any(children): - var["children"] = {n: _qualify(c) for n, c in children.items()} - return var - - def _add_exg_vars(vars: dict) -> dict: - a = subcomponent[:3] - b = subcomponent[:3] - default = f"{a.upper()}6-{b.upper()}6" - return { - "exgtype": { - "name": "exgtype", - "type": "string", - "default": default, - "description": "The exchange type.", - "set_self": True, - }, - "exgmnamea": { - "name": "exgmnamea", - "type": "string", - "description": "The name of the first model in the exchange.", - "set_self": True, - }, - "exgmnameb": { - "name": "exgmnameb", - "type": "string", - "description": "The name of the second model in the exchange.", - "set_self": True, - }, - **vars, - } - - def _variables(vars: dict) -> dict: - """Get the class' member variables.""" - vars_ = vars.copy() - vars_ = _add_exg_vars(vars_) if component == "exg" else vars_ - vars_ = { - name: _qualify(_convert(var)) - for name, var in vars_.items() - # filter components of composites - # since we've inflated the parent - # types in the hierarchy already + vars, _ = definition + _vars = vars.copy() + params = { + name: _convert(var) + for name, var in _vars.items() + # filter composite components + # since we've already inflated + # parent types in the hierarchy if not var.get("in_record", False) } - return vars_ + l, r = name + if l == "sim" and r == "nam": + params = _add_sim_params(params) + elif l == "exg": + params = _add_exg_params(params) + elif r == "nam": + params = _add_mdl_params(params) + else: + params = _add_pkg_params(params) + return {n: p.render() for n, p in params.items()} - def _dfn(vars: dict, meta: list) -> list: + def _dfn(definition: Definition) -> List[Metadata]: """ - Get a list of the class' original definition attributes. + Get a list of the class' original definition attributes, + as an internal/partial reproduction of the DFN contents. Notes ----- @@ -1587,102 +2031,150 @@ def _dfn(vars: dict, meta: list) -> list: Python, consolidating nested types, etc. """ - def _var_dfn(var: dict) -> List[str]: + vars, meta = definition + + def _to_dfn_fmt(var: Variable) -> List[str]: exclude = ["longname", "description"] return [ " ".join([k, v]) for k, v in var.items() if k not in exclude ] return [["header"] + [attr for attr in meta]] + [ - _var_dfn(var) for var in vars.values() + _to_dfn_fmt(var) for var in vars.values() ] return { - "title": _title(component, subcomponent), - "parent": _parent(component, subcomponent), - "component": component, - "subcomponent": subcomponent, - "variables": _variables(definition), - "dfn": _dfn(definition, metadata), + # the input definition's name. + "name": name, + # the corresponding context class name. + "class": name.title, + # the base class the context class inherits from. + "base": _try_qualify(base), + # contexts may be attached to a parent context type. + "parent": { + "name": name.title, + "type": _try_qualify(name.parent), + "kind": name.parent_kind, + }, + # the context's description. header of the docstring. + "description": name.description, + # the context's parameters. inserted into docstring + # and __init__ method, optionally set in __init__. + "parameters": _parameters(definition), + # reproduction of (most of the) original DFN. + "dfn": _dfn(definition), } -def get_src_name(component: str, subcomponent: str): - def _name(): - if component == "sim": - if subcomponent == "nam": - return "simulation" - else: - return subcomponent - elif component == "sln": - return subcomponent - elif component == "exg": - return subcomponent - return f"{component}{subcomponent}" - - return f"mf{_name()}.py" - - -def generate_component(dfn_path: Path, out_path: Path, verbose: bool = False): - comp, sub = dfn_path.stem.split("-") - temp_type = TemplateType.from_pair(comp, sub) - - with open(DFNS_PATH / "common.dfn") as f: - common_vars, _ = load_dfn(f) - - with open(DFNS_PATH / "flopy.dfn") as f: - flopy_vars, _ = load_dfn(f) - - with open(dfn_path, "r") as f: - vars, meta = load_dfn(f) - - # write simulation or package class file - src_path = out_path / get_src_name(comp, sub) - with open(src_path, "w") as f: - context = get_template_context( - component=comp, - subcomponent=sub, - common_vars=common_vars, - flopy_vars=flopy_vars, - definition=vars, - metadata=meta, - ) - template = TEMPLATE_ENV.get_template(f"{temp_type.value}.jinja") +def generate_target( + dfn_name: DefinitionName, + out_path: Path, + common: Variables, + definition: Definition, + subpackages: Definitions, + verbose: bool = False, +): + """ + Generate Python source file(s) from the given input definition. + + Notes + ----- + + Model definitions will produce two files / classes, one for the + model itself and one for its corresponding control file package. + """ + + template = TEMPLATE_ENV.get_template("context.jinja") + context_type = ContextType.from_dfn_name(dfn_name) + context_base = context_type.base + target_path = out_path / dfn_name.target + with open(target_path, "w") as f: + context = { + "context": context_type.value, + **get_template_context( + name=dfn_name, + base=context_base, + common=common, + definition=definition, + references=subpackages, + ), + } source = template.render(**context) if verbose: - print(f"Generating {src_path} from {dfn_path}") + print(f"Generating {target_path}") f.write(source) - if temp_type == TemplateType.Model: - # write model class file - src_path = out_path / get_src_name(comp, "") - with open(src_path, "w") as f: - context = get_template_context( - component=comp, - subcomponent=sub, - common_vars=common_vars, - flopy_vars=flopy_vars, - definition=vars, - metadata=meta, - ) - template = TEMPLATE_ENV.get_template(f"{temp_type.value}.jinja") + # bc models are defined in `-nam.dfn` files, + # we wrote the model class file above, also need + # to write the package class file. + if context_type == ContextType.Model: + _dfn_name = DefinitionName(l=dfn_name.l, r="") + target_path = out_path / _dfn_name.target + with open(target_path, "w") as f: + context = { + "context": "package", + **get_template_context( + name=_dfn_name, + base=MFPackage, + common=common, + definition=definition, + references=subpackages, + ), + } source = template.render(**context) if verbose: - print(f"Generating {src_path} from {dfn_path}") + print(f"Generating {target_path}") f.write(source) -def generate_components(out_path: Path, verbose: bool = False): +def generate_targets(dfn_path: Path, out_path: Path, verbose: bool = False): + """Generate Python source files from the DFN files in the given location.""" + + # find definition files dfn_paths = [ dfn - for dfn in DFNS_PATH.glob("*.dfn") + for dfn in dfn_path.glob("*.dfn") if dfn.stem not in ["common", "flopy"] ] - for dfn_path in dfn_paths: - generate_component(dfn_path, out_path, verbose=verbose) + + # try to load common variables + if not any(p.stem == "common" for p in dfn_paths): + warn("No common input definition file...") + with open(dfn_path / "common.dfn") as f: + common, _ = load_dfn(f) + + # load and generate subpackages first so we can + # identify references to them in other contexts + subpkg_paths = [dfn for dfn in dfn_paths if dfn.stem.startswith("utl")] + subpkgs = dict() + for p in subpkg_paths: + l, r = p.stem.split("-") + name = DefinitionName(l, r) + with open(p) as f: + variables, metadata = load_dfn(f) + definition = (variables, metadata) + subpackage = _try_get_subpkg(metadata) + if subpackage: + subpkgs[r] = subpackage + else: + warn(f"{name} lacks subpackage metadata, only: {metadata}") + generate_target( + name, out_path, common, definition, subpkgs, verbose + ) + + # load and generate the rest of the contexts + context_paths = [p for p in dfn_paths if p not in subpkg_paths] + for p in context_paths: + name = DefinitionName(*p.stem.split("-")) + with open(p) as f: + definition = load_dfn(f) + generate_target( + name, out_path, common, definition, subpkgs, verbose + ) + + # format the generated file run_cmd("ruff", "format", out_path, verbose=verbose) if __name__ == "__main__": - create_packages() - # generate_components(SRCS_PATH) + generate_targets(DFN_PATH, TGT_PATH) diff --git a/flopy/mf6/utils/generate_classes.py b/flopy/mf6/generate_classes.py similarity index 99% rename from flopy/mf6/utils/generate_classes.py rename to flopy/mf6/generate_classes.py index 32c1d6978c..2763503d32 100644 --- a/flopy/mf6/utils/generate_classes.py +++ b/flopy/mf6/generate_classes.py @@ -243,5 +243,5 @@ def cli_main(): if __name__ == "__main__": - """Run command-line with: python -m flopy.mf6.utils.generate_classes""" + """Run command-line with: python -m flopy.mf6.generate_classes""" cli_main() diff --git a/flopy/mf6/templates/attrs.jinja b/flopy/mf6/templates/attrs.jinja index 2b77e65abb..d9f383b100 100644 --- a/flopy/mf6/templates/attrs.jinja +++ b/flopy/mf6/templates/attrs.jinja @@ -1,12 +1,16 @@ {# Class attributes for a component class. #} - {% for name, var in variables.items() if var.type %} - {%- if var.is_iterable or var.is_list or var.is_record %} - {{ var.name }} = ListTemplateGenerator(("{{ component }}6", "{{ subcomponent }}", "{{ var.block }}", "{{ var.name }}")) - {%- elif var.is_array %} - {{ var.name }} = ArrayTemplateGenerator(("{{ component }}6", "{{ subcomponent }}", "{{ var.block }}", "{{ var.name }}")) + {% for name, param in parameters.items() %} + {%- if param.kind == "list" or param.kind == "record" %} + {{ param.name }} = ListTemplateGenerator(("{{ component }}6", "{{ subcomponent }}", "{{ param.block }}", "{{ param.name }}")) + {%- elif param.kind == "array" %} + {{ param.name }} = ArrayTemplateGenerator(("{{ component }}6", "{{ subcomponent }}", "{{ param.block }}", "{{ param.name }}")) {%- endif -%} {%- endfor %} + {%- if context == "model" %} + model_type = {{ component }} + {%- elif context == "package" %} package_abbr = "{{ component }}{{ subcomponent }}" _package_type = "{{ subcomponent }}" dfn_file_name = "{{ component }}-{{ subcomponent }}.dfn" - dfn = {{ dfn|pprint|indent(10) }} \ No newline at end of file + dfn = {{ dfn|pprint|indent(10) }} + {% endif -%} \ No newline at end of file diff --git a/flopy/mf6/templates/package.jinja b/flopy/mf6/templates/context.jinja similarity index 60% rename from flopy/mf6/templates/package.jinja rename to flopy/mf6/templates/context.jinja index 4297f995ab..53de8e174e 100644 --- a/flopy/mf6/templates/package.jinja +++ b/flopy/mf6/templates/context.jinja @@ -4,13 +4,14 @@ import typing import numpy as np from typing import Any, Optional, Tuple, List, Dict, Union, Literal from numpy.typing import NDArray -from .. import mfpackage -from ..data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenerator + +from flopy.mf6.data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenerator +from flopy.mf6.mfpackage import MFPackage from flopy.mf6.mfmodel import MFModel -from flopy.mf6 import MFSimulation +from flopy.mf6 import MFSimulation, MFSimulationBase -class Modflow{{ title }}(mfpackage.MFPackage): +class Modflow{{ class.title() }}({{ base }}): """{% include "docstring.jinja" %}""" {% include "attrs.jinja" %} diff --git a/flopy/mf6/templates/docstring.jinja b/flopy/mf6/templates/docstring.jinja index 687fb29a43..b8ab8a0896 100644 --- a/flopy/mf6/templates/docstring.jinja +++ b/flopy/mf6/templates/docstring.jinja @@ -1,11 +1,11 @@ {# A component class' docstring. #} - Modflow{{ title }} defines a {{ subcomponent }} package within a {{ parent.name }}. + {{ description }} Parameters ---------- {{ parent.name }} : {{ parent.type }} - {{ parent.name.title() }} that this package is a part of. Package is automatically - added to {{ parent.name }} when it is initialized. + {{ parent.kind.title() }} that this package is a part of. Package is automatically + added to the parent {{ parent.kind }} when it is initialized. loading_package : bool Do not set this variable. It is intended for debugging and internal diff --git a/flopy/mf6/templates/init.jinja b/flopy/mf6/templates/init.jinja index a1aa73c189..0328f08130 100644 --- a/flopy/mf6/templates/init.jinja +++ b/flopy/mf6/templates/init.jinja @@ -1,26 +1,67 @@ def __init__( self, - {{ parent.name }}: {{ parent.type }}, - loading_package: bool = False, - {%- for name, var in variables.items() %} - {{ name }}: {{ var.type }} = {{ var.default }}, + {%- for name, param in parameters.items() %} + {{ name }}: {{ param.type }} = {{ param.default }}, {%- endfor %} - filename: Optional[PathLike] = None, - pname: Optional[str] = None, **kwargs, ): +{% if context == "simulation" %} super().__init__( - {{ parent.name }}, "{{ subcomponent }}", filename, pname, loading_package, **kwargs + {%- for name in parameters.keys() %} + {{ name }}={{ name }}, + {%- endfor %}, + **kwargs ) - # set up variables - {%- for name, var in variables.items() if not var.set_self %} - self.{{ name }} = self.build_mfdata("{{ name }}", {{ name }}) - {%- endfor %} - {%- for name, var in variables.items() if var.set_self %} + {%- for name, param in parameters.items() if param.block == "options" %} + self.name_file.{{ name }}.set_data({{ name }}) + self.{{ name }} = self.name_file.{{ name }}, + {%- if param.subpackage is defined %} + self{{ param.subpackage.data_name }} = self._create_package( + "{{ param.subpackage.param_name }}", + self.{{ param.subpackage.data_name }} + ) + {% endif -%} + {% endfor -%} +{% elif context == "model" %} + super().__init__( + model_type="{{ component }}6", + {%- for name in parameters.keys() %} + {{ name }}={{ name }}, + {%- endfor %} + **kwargs, + ) + + {%- for name, param in parameters.items() if param.block == "options" %} + self.name_file.{{ name }}.set_data({{ name }}) + self.{{ name }} = self.name_file.{{ name }}, + {% endfor -%} +{% elif context == "package" %} + super().__init__( + package_type="{{ subcomponent }}", + {%- for name in parameters.keys() %} + {{ name }}={{ name }}, + {%- endfor %} + **kwargs + ) + + {%- for name, param in parameters.items() %} + {%- if param.set_self %} self.{{ name }} = {{ name }} + {% else %} + self.{{ name }} = self.build_mfdata("{{ name }}", {{ name }}) + {% endif -%} + {%- if param.subpackage is defined %} + self._{{ param.subpackage.abbr }}_package = self.build_child_package( + "{{ param.subpackage.abbrev }}", + {{ param.subpackage.data_name }}, + "{{ param.subpackage.param_name }}", + self.{{ param.subpackage.record_name }} + ) + {% endif -%} {%- endfor %} {%- if component == "exg" %} simulation.register_exchange_file(self) {% endif -%} - self._init_complete = True \ No newline at end of file + self._init_complete = True +{% endif %} \ No newline at end of file diff --git a/flopy/mf6/templates/params.jinja b/flopy/mf6/templates/params.jinja index 32a20005bd..3be82750b4 100644 --- a/flopy/mf6/templates/params.jinja +++ b/flopy/mf6/templates/params.jinja @@ -1,9 +1,9 @@ -{% for var in variables.values() recursive %} - {% if loop.depth > 1 %}* {% endif %}{{ var.name }} : {{ var.type }} -{%- if var.description is defined and not var.is_choice %} -{{ var.description|wordwrap|indent(4 + (loop.depth * 4), first=True) }} +{% for param in parameters.values() recursive %} + {% if loop.depth > 1 %}* {% endif %}{{ param.name }} : {{ param.type }} +{%- if param.description is defined and not param.is_choice %} +{{ param.description|wordwrap|indent(4 + (loop.depth * 4), first=True) }} {%- endif %} -{%- if var.children is defined -%} -{{ loop(var.children.values())|indent(4) }} +{%- if param.children is defined -%} +{{ loop(param.children.values())|indent(4) }} {%- endif %} {% endfor %} \ No newline at end of file diff --git a/flopy/mf6/utils/__init__.py b/flopy/mf6/utils/__init__.py index dd19d3a5d2..92605c7722 100644 --- a/flopy/mf6/utils/__init__.py +++ b/flopy/mf6/utils/__init__.py @@ -1,6 +1,4 @@ -from . import createpackages from .binarygrid_util import MfGrdFile -from .generate_classes import generate_classes from .lakpak_utils import get_lak_connections from .mfsimlistfile import MfSimulationList from .model_splitter import Mf6Splitter diff --git a/flopy/mf6/utils/dfn.py b/flopy/mf6/utils/dfn.py deleted file mode 100644 index 12bf1506bd..0000000000 --- a/flopy/mf6/utils/dfn.py +++ /dev/null @@ -1,46 +0,0 @@ -from typing import Dict, List, Tuple, Union - -Scalar = Union[bool, int, float, str] -Definition = Dict[str, Dict[str, Scalar]] - - -def load_dfn(f) -> Tuple[Definition, List[str]]: - """ - Load an input definition file. Returns a tuple containing - a dictionary variables and a list of metadata attributes. - """ - meta = list() - vars = dict() - var = dict() - - for line in f: - # remove whitespace/etc from the line - line = line.strip() - - # record flopy metadata attributes but - # skip all other comment lines - if line.startswith("#"): - _, sep, tail = line.partition("flopy") - if sep == "flopy": - meta.append(tail.strip()) - continue - - # if we hit a newline and the parameter dict - # is nonempty, we've reached the end of its - # block of attributes - if not any(line): - if any(var): - vars[var["name"]] = var - var = dict() - continue - - # split the attribute's key and value and - # store it in the parameter dictionary - key, _, value = line.partition(" ") - var[key] = value - - # add the final parameter - if any(var): - vars[var["name"]] = var - - return vars, meta From 2713059fe6f2def430d7c60a6a21c80e92fc391d Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 1 Oct 2024 16:17:52 -0400 Subject: [PATCH 12/46] subpackages working? much cleanup --- autotest/test_createpackages.py | 93 +- flopy/mf6/createpackages.py | 2180 ----------------- flopy/mf6/templates/attrs.jinja | 16 - flopy/mf6/templates/docstring.jinja | 23 - flopy/mf6/templates/init.jinja | 67 - flopy/mf6/templates/model.jinja | 141 -- flopy/mf6/templates/params.jinja | 9 - flopy/mf6/templates/simulation.jinja | 99 - flopy/mf6/utils/createpackages.py | 1457 +++++++++++ flopy/mf6/{ => utils}/generate_classes.py | 0 flopy/mf6/utils/templates/attrs.jinja | 15 + .../templates/context.py.jinja} | 10 +- flopy/mf6/utils/templates/docstring.jinja | 12 + .../utils/templates/docstring_methods.jinja | 13 + .../utils/templates/docstring_params.jinja | 9 + flopy/mf6/utils/templates/init.jinja | 72 + flopy/mf6/utils/templates/load.jinja | 58 + 17 files changed, 1679 insertions(+), 2595 deletions(-) delete mode 100644 flopy/mf6/createpackages.py delete mode 100644 flopy/mf6/templates/attrs.jinja delete mode 100644 flopy/mf6/templates/docstring.jinja delete mode 100644 flopy/mf6/templates/init.jinja delete mode 100644 flopy/mf6/templates/model.jinja delete mode 100644 flopy/mf6/templates/params.jinja delete mode 100644 flopy/mf6/templates/simulation.jinja create mode 100644 flopy/mf6/utils/createpackages.py rename flopy/mf6/{ => utils}/generate_classes.py (100%) create mode 100644 flopy/mf6/utils/templates/attrs.jinja rename flopy/mf6/{templates/context.jinja => utils/templates/context.py.jinja} (73%) create mode 100644 flopy/mf6/utils/templates/docstring.jinja create mode 100644 flopy/mf6/utils/templates/docstring_methods.jinja create mode 100644 flopy/mf6/utils/templates/docstring_params.jinja create mode 100644 flopy/mf6/utils/templates/init.jinja create mode 100644 flopy/mf6/utils/templates/load.jinja diff --git a/autotest/test_createpackages.py b/autotest/test_createpackages.py index 7ec57b42c8..6d520886e3 100644 --- a/autotest/test_createpackages.py +++ b/autotest/test_createpackages.py @@ -2,84 +2,65 @@ from modflow_devtools.misc import run_cmd from autotest.conftest import get_project_root_path -from flopy.mf6.createpackages import ( - TEMPLATE_ENV, - ContextType, - DefinitionName, - generate_targets, - get_template_context, +from flopy.mf6.utils.createpackages import ( + DfnName, load_dfn, + make_all, + make_context, + make_targets, ) -from flopy.mf6.mfpackage import MFPackage PROJ_ROOT = get_project_root_path() DFN_PATH = PROJ_ROOT / "flopy" / "mf6" / "data" / "dfn" -DFNS = [ - dfn +DFN_NAMES = [ + dfn.stem for dfn in DFN_PATH.glob("*.dfn") if dfn.stem not in ["common", "flopy"] ] -@pytest.mark.parametrize("dfn", DFNS) -def test_load_dfn(dfn): - dfn_path = DFN_PATH / dfn +@pytest.mark.parametrize("dfn_name", DFN_NAMES) +def test_load_dfn(dfn_name): + dfn_path = DFN_PATH / f"{dfn_name}.dfn" with open(dfn_path, "r") as f: - definition = load_dfn(f) + dfn = load_dfn(f, name=DfnName(*dfn_name.split("-"))) -# only test packages for which we know the -# expected number of consolidated variables @pytest.mark.parametrize( - "dfn, n_flat, n_params", [("gwf-ic", 2, 6), ("prt-prp", 40, 22)] + "dfn_name, n_flat, n_params", [("gwf-ic", 2, 6), ("prt-prp", 40, 22)] ) -def test_get_template_context(dfn, n_flat, n_params): - dfn_name = DefinitionName(*dfn.split("-")) - +def test_make_context(dfn_name, n_flat, n_params): with open(DFN_PATH / "common.dfn") as f: - common, _ = load_dfn(f) - - with open(DFN_PATH / f"{dfn}.dfn") as f: - definition = load_dfn(f) + common = load_dfn(f) - context = get_template_context( - dfn_name, - MFPackage, - common, - definition, - ) - assert context["name"] == dfn_name - assert len(context["parameters"]) == n_params - assert len(context["dfn"]) == n_flat + 1 # +1 for metadata + with open(DFN_PATH / f"{dfn_name}.dfn") as f: + dfn_name = DfnName(*dfn_name.split("-")) + dfn = load_dfn(f, name=dfn_name) + ctx_name = dfn_name.contexts[0] + context = make_context(ctx_name, dfn, common=common) + assert len(dfn_name.contexts) == 1 + assert len(context.variables) == n_params + assert len(context.metadata) == n_flat + 1 # +1 for metadata -@pytest.mark.parametrize("dfn", [dfn.stem for dfn in DFNS]) -def test_render_template(dfn, function_tmpdir): - dfn_name = DefinitionName(*dfn.split("-")) - context_type = ContextType.from_dfn_name(dfn_name) - template = TEMPLATE_ENV.get_template("context.jinja") +@pytest.mark.parametrize("dfn_name", DFN_NAMES) +def test_make_targets(dfn_name, function_tmpdir): with open(DFN_PATH / "common.dfn") as f: - common, _ = load_dfn(f) + common = load_dfn(f) - with open(DFN_PATH / f"{dfn}.dfn", "r") as f: - definition = load_dfn(f) + with open(DFN_PATH / f"{dfn_name}.dfn", "r") as f: + dfn_name = DfnName(*dfn_name.split("-")) + dfn = load_dfn(f, name=dfn_name) - context = { - "context": context_type.value, - **get_template_context( - dfn_name, - context_type.base, - common, - definition, - ), - } - source = template.render(**context) - source_path = function_tmpdir / dfn_name.target - with open(source_path, "w") as f: - f.write(source) - run_cmd("ruff", "format", source_path, verbose=True) + make_targets(dfn, function_tmpdir, common=common) + for ctx_name in dfn_name.contexts: + run_cmd("ruff", "format", function_tmpdir, verbose=True) + run_cmd("ruff", "check", "--fix", function_tmpdir, verbose=True) + assert (function_tmpdir / ctx_name.target).is_file() -def test_generate_components(function_tmpdir): - generate_targets(DFN_PATH, function_tmpdir, verbose=True) +def test_make_all(function_tmpdir): + make_all(DFN_PATH, function_tmpdir, verbose=True) + run_cmd("ruff", "format", function_tmpdir, verbose=True) + run_cmd("ruff", "check", "--fix", function_tmpdir, verbose=True) diff --git a/flopy/mf6/createpackages.py b/flopy/mf6/createpackages.py deleted file mode 100644 index cd8312c1e5..0000000000 --- a/flopy/mf6/createpackages.py +++ /dev/null @@ -1,2180 +0,0 @@ -""" -createpackages.py is a utility script that reads in the file definition -metadata in the .dfn files and creates the package classes in the modflow -folder. Run this script any time changes are made to the .dfn files. - -To create a new package that is part of an existing model, first create a new -dfn file for the package in the mf6/data/dfn folder. -1) Follow the file naming convention -.dfn. -2) Run this script (createpackages.py), and check in your new dfn file, and - the package class and updated __init__.py that createpackages.py created. - -A subpackage is a package referenced by another package (vs being referenced -in the name file). The tas, ts, and obs packages are examples of subpackages. -There are a few additional steps required when creating a subpackage -definition file. First, verify that the parent package's dfn file has a file -record for the subpackage to the option block. For example, for the time -series package the file record definition starts with: - - block options - name ts_filerecord - type record ts6 filein ts6_filename - -Verify that the same naming convention is followed as the example above, -specifically: - - name _filerecord - record 6 filein 6_filename - -Next, create the child package definition file in the mf6/data/dfn folder -following the naming convention above. - -When your child package is ready for release follow the same procedure as -other packages along with these a few additional steps required for -subpackages. - -At the top of the child dfn file add two lines describing how the parent and -child packages are related. The first line determines how the subpackage is -linked to the package: - -# flopy subpackage - - -* Parent record is the MF6 record name of the filerecord in parent package - that references the child packages file name -* Abbreviation is the short abbreviation of the new subclass -* Child data is the name of the child class data that can be passed in as - parameter to the parent class. Passing in this parameter to the parent class - automatically creates the child class with the data provided. -* Data name is the parent class parameter name that automatically creates the - child class with the data provided. - -The example below is the first line from the ts subpackage dfn: - -# flopy subpackage ts_filerecord ts timeseries timeseries - -The second line determines the variable name of the subpackage's parent and -the type of parent (the parent package's object oriented parent): - -# flopy parent_name_type - - -An example below is the second line in the ts subpackage dfn: - -# flopy parent_name_type parent_package MFPackage - -There are three possible types (or combination of them) that can be used for -"parent package type", MFPackage, MFModel, and MFSimulation. If a package -supports multiple types of parents (for example, it can be either in the model -namefile or in a package, like the obs package), include all the types -supported, separating each type with a / (MFPackage/MFModel). - -To create a new type of model choose a unique three letter model abbreviation -("gwf", "gwt", ...). Create a name file dfn with the naming convention --nam.dfn. The name file must have only an options and packages -block (see gwf-nam.dfn as an example). Create a new dfn file for each of the -packages in your new model, following the naming convention described above. - -When your model is ready for release make sure all the dfn files are in the -flopy/mf6/data/dfn folder, run createpackages.py, and check in your new dfn -files, the package classes, and updated init.py that createpackages.py created. - -""" - -import collections -import datetime -import os -import textwrap -from abc import ABC, abstractmethod -from dataclasses import asdict, dataclass, replace -from enum import Enum -from keyword import kwlist -from os import PathLike -from pathlib import Path -from typing import ( - Any, - Dict, - ForwardRef, - Iterable, - List, - Literal, - NamedTuple, - Optional, - Tuple, - TypedDict, - Union, - get_args, - get_origin, -) -from warnings import warn - -import numpy as np -from jinja2 import Environment, PackageLoader -from modflow_devtools.misc import run_cmd -from numpy.typing import ArrayLike, NDArray - -from flopy.mf6 import MFSimulation - -# keep below as absolute imports -from flopy.mf6.data import mfdatautil, mfstructure -from flopy.mf6.mfmodel import MFModel -from flopy.mf6.mfpackage import MFPackage -from flopy.mf6.mfsimbase import MFSimulationBase -from flopy.utils import datautil - - -class PackageLevel(Enum): - sim_level = 0 - model_level = 1 - - -def build_doc_string(param_name, param_type, param_desc, indent): - return f"{indent}{param_name} : {param_type}\n{indent * 2}* {param_desc}" - - -def generator_type(data_type): - if ( - data_type == mfstructure.DataType.scalar_keyword - or data_type == mfstructure.DataType.scalar - ): - # regular scalar - return "ScalarTemplateGenerator" - elif ( - data_type == mfstructure.DataType.scalar_keyword_transient - or data_type == mfstructure.DataType.scalar_transient - ): - # transient scalar - return "ScalarTemplateGenerator" - elif data_type == mfstructure.DataType.array: - # array - return "ArrayTemplateGenerator" - elif data_type == mfstructure.DataType.array_transient: - # transient array - return "ArrayTemplateGenerator" - elif data_type == mfstructure.DataType.list: - # list - return "ListTemplateGenerator" - elif ( - data_type == mfstructure.DataType.list_transient - or data_type == mfstructure.DataType.list_multiple - ): - # transient or multiple list - return "ListTemplateGenerator" - - -def clean_class_string(name): - if len(name) > 0: - clean_string = name.replace(" ", "_") - clean_string = clean_string.replace("-", "_") - version = mfstructure.MFStructure().get_version_string() - # FIX: remove all numbers - if clean_string[-1] == version: - clean_string = clean_string[:-1] - return clean_string - return name - - -def build_dfn_string(dfn_list, header, package_abbr, flopy_dict): - dfn_string = " dfn = [" - line_length = len(dfn_string) - leading_spaces = " " * line_length - first_di = True - - # process header - dfn_string = f'{dfn_string}\n{leading_spaces}["header", ' - for key, value in header.items(): - if key == "multi-package": - dfn_string = f'{dfn_string}\n{leading_spaces} "multi-package", ' - if key == "package-type": - dfn_string = ( - f'{dfn_string}\n{leading_spaces} "package-type ' f'{value}"' - ) - - # process solution packages - if package_abbr in flopy_dict["solution_packages"]: - model_types = '", "'.join( - flopy_dict["solution_packages"][package_abbr] - ) - dfn_string = ( - f"{dfn_string}\n{leading_spaces} " - f'["solution_package", "{model_types}"], ' - ) - dfn_string = f"{dfn_string}],\n{leading_spaces}" - - # process all data items - for data_item in dfn_list: - line_length += 1 - if not first_di: - dfn_string = f"{dfn_string},\n{leading_spaces}" - line_length = len(leading_spaces) - else: - first_di = False - dfn_string = f"{dfn_string}[" - first_line = True - # process each line in a data item - for line in data_item: - line = line.strip() - # do not include the description of longname - if not line.lower().startswith( - "description" - ) and not line.lower().startswith("longname"): - line = line.replace('"', "'") - line_length += len(line) + 4 - if not first_line: - dfn_string = f"{dfn_string}," - if line_length < 77: - # added text fits on the current line - if first_line: - dfn_string = f'{dfn_string}"{line}"' - else: - dfn_string = f'{dfn_string} "{line}"' - else: - # added text does not fit on the current line - line_length = len(line) + len(leading_spaces) + 2 - if line_length > 79: - # added text too long to fit on a single line, wrap - # text as needed - line = f'"{line}"' - lines = textwrap.wrap( - line, - 75 - len(leading_spaces), - drop_whitespace=True, - ) - lines[0] = f"{leading_spaces} {lines[0]}" - line_join = f' "\n{leading_spaces} "' - dfn_string = f"{dfn_string}\n{line_join.join(lines)}" - else: - dfn_string = f'{dfn_string}\n{leading_spaces} "{line}"' - first_line = False - - dfn_string = f"{dfn_string}]" - dfn_string = f"{dfn_string}]" - return dfn_string - - -def create_init_var(clean_ds_name, data_structure_name, init_val=None): - if init_val is None: - init_val = clean_ds_name - - init_var = f" self.{clean_ds_name} = self.build_mfdata(" - leading_spaces = " " * len(init_var) - if len(init_var) + len(data_structure_name) + 2 > 79: - second_line = f'\n "{data_structure_name}",' - if len(second_line) + len(clean_ds_name) + 2 > 79: - init_var = f"{init_var}{second_line}\n {init_val})" - else: - init_var = f"{init_var}{second_line} {init_val})" - else: - init_var = f'{init_var}"{data_structure_name}",' - if len(init_var) + len(clean_ds_name) + 2 > 79: - init_var = f"{init_var}\n{leading_spaces}{init_val})" - else: - init_var = f"{init_var} {init_val})" - return init_var - - -def create_basic_init(clean_ds_name): - return f" self.{clean_ds_name} = {clean_ds_name}\n" - - -def create_property(clean_ds_name): - return f" {clean_ds_name} = property(get_{clean_ds_name}, set_{clean_ds_name})" - - -def format_var_list(base_string, var_list, is_tuple=False): - if is_tuple: - base_string = f"{base_string}(" - extra_chars = 4 - else: - extra_chars = 2 - line_length = len(base_string) - leading_spaces = " " * line_length - # determine if any variable name is too long to fit - for item in var_list: - if line_length + len(item) + extra_chars > 80: - leading_spaces = " " - base_string = f"{base_string}\n{leading_spaces}" - line_length = len(leading_spaces) - break - - for index, item in enumerate(var_list): - if is_tuple: - item = f"'{item}'" - if index == len(var_list) - 1: - next_var_str = item - else: - next_var_str = f"{item}," - line_length += len(item) + extra_chars - if line_length > 80: - base_string = f"{base_string}\n{leading_spaces}{next_var_str}" - else: - if base_string[-1] == ",": - base_string = f"{base_string} " - base_string = f"{base_string}{next_var_str}" - if is_tuple: - return f"{base_string}))" - else: - return f"{base_string})" - - -def create_package_init_var( - parameter_name, package_abbr, data_name, clean_ds_name -): - one_line = ( - f" self._{package_abbr}_package = self.build_child_package(" - ) - one_line_b = f'"{package_abbr}", {parameter_name},' - leading_spaces = " " * len(one_line) - two_line = f'\n{leading_spaces}"{data_name}",' - three_line = f"\n{leading_spaces}self._{clean_ds_name})" - return f"{one_line}{one_line_b}{two_line}{three_line}" - - -def add_var( - init_vars, - class_vars, - options_param_list, - init_param_list, - package_properties, - doc_string, - data_structure_dict, - default_value, - name, - python_name, - description, - path, - data_type, - basic_init=False, - construct_package=None, - construct_data=None, - parameter_name=None, - set_param_list=None, - mf_nam=False, -): - if set_param_list is None: - set_param_list = [] - clean_ds_name = datautil.clean_name(python_name) - if construct_package is None: - # add variable initialization lines - if basic_init: - init_vars.append(create_basic_init(clean_ds_name)) - else: - init_vars.append(create_init_var(clean_ds_name, name)) - # add to parameter list - if default_value is None: - default_value = "None" - init_param_list.append(f"{clean_ds_name}={default_value}") - if path is not None and "options" in path: - options_param_list.append(f"{clean_ds_name}={default_value}") - # add to set parameter list - set_param_list.append(f"{clean_ds_name}={clean_ds_name}") - else: - clean_parameter_name = datautil.clean_name(parameter_name) - # init hidden variable - init_vars.append(create_init_var(f"_{clean_ds_name}", name, "None")) - if mf_nam: - options_param_list.append( - [f"{parameter_name}_data=None", parameter_name] - ) - else: - # init child package - init_vars.append( - create_package_init_var( - clean_parameter_name, - construct_package, - construct_data, - clean_ds_name, - ) - ) - # add to parameter list - init_param_list.append(f"{clean_parameter_name}=None") - # add to set parameter list - set_param_list.append( - f"{clean_parameter_name}={clean_parameter_name}" - ) - - package_properties.append(create_property(clean_ds_name)) - doc_string.add_parameter(description, model_parameter=True) - data_structure_dict[python_name] = 0 - if class_vars is not None: - gen_type = generator_type(data_type) - if gen_type != "ScalarTemplateGenerator": - new_class_var = f" {clean_ds_name} = {gen_type}(" - class_vars.append(format_var_list(new_class_var, path, True)) - return gen_type - return None - - -def build_init_string( - init_string, init_param_list, whitespace=" " -): - line_chars = len(init_string) - for index, param in enumerate(init_param_list): - if isinstance(param, list): - param = param[0] - if index + 1 < len(init_param_list): - line_chars += len(param) + 2 - else: - line_chars += len(param) + 3 - if line_chars > 79: - if len(param) + len(whitespace) + 1 > 79: - # try to break apart at = sign - param_list = param.split("=") - if len(param_list) == 2: - init_string = "{},\n{}{}=\n{}{}".format( - init_string, - whitespace, - param_list[0], - whitespace, - param_list[1], - ) - line_chars = len(param_list[1]) + len(whitespace) + 1 - continue - init_string = f"{init_string},\n{whitespace}{param}" - line_chars = len(param) + len(whitespace) + 1 - else: - init_string = f"{init_string}, {param}" - return f"{init_string}):\n" - - -def build_model_load(model_type): - model_load_c = ( - " Methods\n -------\n" - " load : (simulation : MFSimulationData, model_name : " - "string,\n namfile : string, " - "version : string, exe_name : string,\n model_ws : " - "string, strict : boolean) : MFSimulation\n" - " a class method that loads a model from files" - '\n """' - ) - - model_load = ( - " @classmethod\n def load(cls, simulation, structure, " - "modelname='NewModel',\n " - "model_nam_file='modflowtest.nam', version='mf6',\n" - " exe_name='mf6', strict=True, " - "model_rel_path='.',\n" - " load_only=None):\n " - "return mfmodel.MFModel.load_base(cls, simulation, structure, " - "modelname,\n " - "model_nam_file, '{}6', version,\n" - " exe_name, strict, " - "model_rel_path,\n" - " load_only)" - "\n".format(model_type) - ) - return model_load, model_load_c - - -def build_sim_load(): - sim_load_c = ( - " Methods\n -------\n" - " load : (sim_name : str, version : " - "string,\n exe_name : str or PathLike, " - "sim_ws : str or PathLike, strict : bool,\n verbosity_level : " - "int, load_only : list, verify_data : bool,\n " - "write_headers : bool, lazy_io : bool, use_pandas : bool,\n " - ") : MFSimulation\n" - " a class method that loads a simulation from files" - '\n """' - ) - - sim_load = ( - " @classmethod\n def load(cls, sim_name='modflowsim', " - "version='mf6',\n " - "exe_name: Union[str, os.PathLike] = 'mf6',\n " - "sim_ws: Union[str, os.PathLike] = os.curdir,\n " - "strict=True, verbosity_level=1, load_only=None,\n " - "verify_data=False, write_headers=True,\n " - "lazy_io=False, use_pandas=True):\n " - "return mfsimbase.MFSimulationBase.load(cls, sim_name, version, " - "\n " - "exe_name, sim_ws, strict,\n" - " verbosity_level, " - "load_only,\n " - "verify_data, write_headers, " - "\n lazy_io, use_pandas)" - "\n" - ) - return sim_load, sim_load_c - - -def build_model_init_vars(param_list): - init_var_list = [] - # build set data calls - for param in param_list: - if not isinstance(param, list): - param_parts = param.split("=") - init_var_list.append( - f" self.name_file.{param_parts[0]}.set_data({param_parts[0]})" - ) - init_var_list.append("") - # build attributes - for param in param_list: - if isinstance(param, list): - pkg_name = param[1] - param_parts = param[0].split("=") - init_var_list.append( - f" self.{param_parts[0]} = " - f"self._create_package('{pkg_name}', {param_parts[0]})" - ) - else: - param_parts = param.split("=") - init_var_list.append( - f" self.{param_parts[0]} = self.name_file.{param_parts[0]}" - ) - - return "\n".join(init_var_list) - - -def create_packages(): - indent = " " - init_string_def = " def __init__(self" - - # load JSON file - file_structure = mfstructure.MFStructure(load_from_dfn_files=True) - sim_struct = file_structure.sim_struct - - # assemble package list of buildable packages - package_list = [] - for package in sim_struct.utl_struct_objs.values(): - # add utility packages to list - package_list.append( - ( - package, - PackageLevel.model_level, - "utl", - package.dfn_list, - package.file_type, - package.header, - ) - ) - package_list.append( - ( - sim_struct.name_file_struct_obj, - PackageLevel.sim_level, - "", - sim_struct.name_file_struct_obj.dfn_list, - sim_struct.name_file_struct_obj.file_type, - sim_struct.name_file_struct_obj.header, - ) - ) - for package in sim_struct.package_struct_objs.values(): - # add simulation level package to list - package_list.append( - ( - package, - PackageLevel.sim_level, - "", - package.dfn_list, - package.file_type, - package.header, - ) - ) - for model_key, model in sim_struct.model_struct_objs.items(): - package_list.append( - ( - model.name_file_struct_obj, - PackageLevel.model_level, - model_key, - model.name_file_struct_obj.dfn_list, - model.name_file_struct_obj.file_type, - model.name_file_struct_obj.header, - ) - ) - for package in model.package_struct_objs.values(): - package_list.append( - ( - package, - PackageLevel.model_level, - model_key, - package.dfn_list, - package.file_type, - package.header, - ) - ) - - util_path, tail = os.path.split(os.path.realpath(__file__)) - init_file = open( - os.path.join(util_path, "..", "modflow", "__init__.py"), - "w", - newline="\n", - ) - init_file.write("from .mfsimulation import MFSimulation # isort:skip\n") - - nam_import_string = ( - "from .. import mfmodel\nfrom ..data.mfdatautil " - "import ArrayTemplateGenerator, ListTemplateGenerator" - ) - - # loop through packages list - init_file_imports = [] - flopy_dict = file_structure.flopy_dict - for package in package_list: - data_structure_dict = {} - package_properties = [] - init_vars = [] - init_param_list = [] - options_param_list = [] - set_param_list = [] - class_vars = [] - template_gens = [] - - package_abbr = clean_class_string( - f"{clean_class_string(package[2])}{package[0].file_type}" - ).lower() - dfn_string = build_dfn_string( - package[3], package[5], package_abbr, flopy_dict - ) - package_name = clean_class_string( - "{}{}{}".format( - clean_class_string(package[2]), - package[0].file_prefix, - package[0].file_type, - ) - ).lower() - if package[0].description: - doc_string = mfdatautil.MFDocString(package[0].description) - else: - if package[2]: - package_container_text = f" within a {package[2]} model" - else: - package_container_text = "" - ds = "Modflow{} defines a {} package{}.".format( - package_name.title(), - package[0].file_type, - package_container_text, - ) - if package[0].file_type == "mvr": - # mvr package warning - if package[2]: - ds = ( - "{} This package\n can only be used to move " - "water between packages within a single model." - "\n To move water between models use ModflowMvr" - ".".format(ds) - ) - else: - ds = ( - "{} This package can only be used to move\n " - "water between two different models. To move " - "water between two packages\n in the same " - 'model use the "model level" mover package (ex. ' - "ModflowGwfmvr).".format(ds) - ) - - doc_string = mfdatautil.MFDocString(ds) - - if package[0].dfn_type == mfstructure.DfnType.exch_file: - exgtype = ( - f'"{package_abbr[0:3].upper()}6-{package_abbr[3:].upper()}6"' - ) - - add_var( - init_vars, - None, - options_param_list, - init_param_list, - package_properties, - doc_string, - data_structure_dict, - exgtype, - "exgtype", - "exgtype", - build_doc_string( - "exgtype", - "", - "is the exchange type (GWF-GWF or GWF-GWT).", - indent, - ), - None, - None, - True, - ) - add_var( - init_vars, - None, - options_param_list, - init_param_list, - package_properties, - doc_string, - data_structure_dict, - None, - "exgmnamea", - "exgmnamea", - build_doc_string( - "exgmnamea", - "", - "is the name of the first model that is " - "part of this exchange.", - indent, - ), - None, - None, - True, - ) - add_var( - init_vars, - None, - options_param_list, - init_param_list, - package_properties, - doc_string, - data_structure_dict, - None, - "exgmnameb", - "exgmnameb", - build_doc_string( - "exgmnameb", - "", - "is the name of the second model that is " - "part of this exchange.", - indent, - ), - None, - None, - True, - ) - init_vars.append( - " simulation.register_exchange_file(self)\n" - ) - - # loop through all blocks - for block in package[0].blocks.values(): - for data_structure in block.data_structures.values(): - # only create one property for each unique data structure name - if data_structure.name not in data_structure_dict: - mf_sim = ( - "parent_name_type" in package[0].header - and package[0].header["parent_name_type"][1] - == "MFSimulation" - ) - mf_nam = package[0].file_type == "nam" - if ( - data_structure.construct_package is not None - and not mf_sim - and not mf_nam - ): - c_pkg = data_structure.construct_package - else: - c_pkg = None - tg = add_var( - init_vars, - class_vars, - options_param_list, - init_param_list, - package_properties, - doc_string, - data_structure_dict, - data_structure.default_value, - data_structure.name, - data_structure.python_name, - data_structure.get_doc_string(79, indent, indent), - data_structure.path, - data_structure.get_datatype(), - False, - # c_pkg, - data_structure.construct_package, - data_structure.construct_data, - data_structure.parameter_name, - set_param_list, - mf_nam, - ) - if tg is not None and tg not in template_gens: - template_gens.append(tg) - - import_string = "from .. import mfpackage" - if template_gens: - import_string += "\nfrom ..data.mfdatautil import " - import_string += ", ".join(sorted(template_gens)) - # add extra docstrings for additional variables - doc_string.add_parameter( - " filename : String\n File name for this package." - ) - doc_string.add_parameter( - " pname : String\n Package name for this package." - ) - doc_string.add_parameter( - " parent_file : MFPackage\n " - "Parent package file that references this " - "package. Only needed for\n utility " - "packages (mfutl*). For example, mfutllaktab " - "package must have \n a mfgwflak " - "package parent_file." - ) - - # build package builder class string - init_vars.append(" self._init_complete = True") - init_vars = "\n".join(init_vars) - package_short_name = clean_class_string(package[0].file_type).lower() - class_def_string = "class Modflow{}(mfpackage.MFPackage):\n".format( - package_name.title() - ) - class_def_string = class_def_string.replace("-", "_") - class_var_string = ( - '{}\n package_abbr = "{}"\n _package_type = ' - '"{}"\n dfn_file_name = "{}"' - "\n".format( - "\n".join(class_vars), - package_abbr, - package[4], - package[0].dfn_file_name, - ) - ) - init_string_full = init_string_def - init_string_sim = f"{init_string_def}, simulation" - # add variables to init string - doc_string.add_parameter( - " loading_package : bool\n " - "Do not set this parameter. It is intended " - "for debugging and internal\n " - "processing purposes only.", - beginning_of_list=True, - ) - if "parent_name_type" in package[0].header: - init_var = package[0].header["parent_name_type"][0] - parent_type = package[0].header["parent_name_type"][1] - elif package[1] == PackageLevel.sim_level: - init_var = "simulation" - parent_type = "MFSimulation" - else: - init_var = "model" - parent_type = "MFModel" - doc_string.add_parameter( - f" {init_var} : {parent_type}\n " - f"{init_var.capitalize()} that this package is a part " - "of. Package is automatically\n " - f"added to {init_var} when it is " - "initialized.", - beginning_of_list=True, - ) - init_string_full = ( - f"{init_string_full}, {init_var}, loading_package=False" - ) - init_param_list.append("filename=None") - init_param_list.append("pname=None") - init_param_list.append("**kwargs") - init_string_full = build_init_string(init_string_full, init_param_list) - - # build init code - parent_init_string = " super().__init__(" - spaces = " " * len(parent_init_string) - parent_init_string = ( - '{}{}, "{}", filename, pname,\n{}' - "loading_package, **kwargs)\n\n" - " # set up variables".format( - parent_init_string, init_var, package_short_name, spaces - ) - ) - local_datetime = datetime.datetime.now(datetime.timezone.utc) - comment_string = ( - "# DO NOT MODIFY THIS FILE DIRECTLY. THIS FILE " - "MUST BE CREATED BY\n# mf6/createpackages.py\n" - "# FILE created on {} UTC".format( - local_datetime.strftime("%B %d, %Y %H:%M:%S") - ) - ) - # assemble full package string - package_string = "{}\n{}\n\n\n{}{}\n{}\n{}\n\n{}{}\n{}\n".format( - comment_string, - import_string, - class_def_string, - doc_string.get_doc_string(), - class_var_string, - dfn_string, - init_string_full, - parent_init_string, - init_vars, - ) - - # open new Packages file - pb_file = open( - os.path.join(util_path, "..", "modflow", f"mf{package_name}.py"), - "w", - newline="\n", - ) - pb_file.write(package_string) - if ( - package[0].sub_package - and package_abbr != "utltab" - and ( - "parent_name_type" not in package[0].header - or package[0].header["parent_name_type"][1] != "MFSimulation" - ) - ): - set_param_list.append("filename=filename") - set_param_list.append("pname=pname") - set_param_list.append("child_builder_call=True") - whsp_1 = " " - whsp_2 = " " - - file_prefix = package[0].dfn_file_name[0:3] - chld_doc_string = ( - ' """\n {}Packages is a container ' - "class for the Modflow{} class.\n\n " - "Methods\n ----------" - "\n".format(package_name.title(), package_name.title()) - ) - - # write out child packages class - chld_cls = ( - "\n\nclass {}Packages(mfpackage.MFChildPackage" "s):\n".format( - package_name.title() - ) - ) - chld_var = ( - f" package_abbr = " - f'"{package_name.title().lower()}packages"\n\n' - ) - chld_init = " def initialize(self" - chld_init = build_init_string( - chld_init, init_param_list[:-1], whsp_1 - ) - init_pkg = "\n self.init_package(new_package, filename)" - params_init = ( - " new_package = Modflow" - f"{package_name.title()}(self._cpparent" - ) - params_init = build_init_string( - params_init, set_param_list, whsp_2 - ) - chld_doc_string = ( - "{} initialize\n Initializes a new " - "Modflow{} package removing any sibling " - "child\n packages attached to the same " - "parent package. See Modflow{} init\n " - " documentation for definition of " - "parameters.\n".format( - chld_doc_string, package_name.title(), package_name.title() - ) - ) - - chld_appn = "" - params_appn = "" - append_pkg = "" - if package_abbr != "utlobs": # Hard coded obs no multi-pkg support - chld_appn = "\n\n def append_package(self" - chld_appn = build_init_string( - chld_appn, init_param_list[:-1], whsp_1 - ) - append_pkg = ( - "\n self._append_package(new_package, filename)" - ) - params_appn = ( - " new_package = Modflow" - f"{file_prefix.capitalize()}" - f"{package_short_name}(self._cpparent" - ) - params_appn = build_init_string( - params_appn, set_param_list, whsp_2 - ) - chld_doc_string = ( - "{} append_package\n Adds a " - "new Modflow{}{} package to the container." - " See Modflow{}{}\n init " - "documentation for definition of " - "parameters.\n".format( - chld_doc_string, - file_prefix.capitalize(), - package_short_name, - file_prefix.capitalize(), - package_short_name, - ) - ) - chld_doc_string = f'{chld_doc_string} """\n' - packages_str = "{}{}{}{}{}{}{}{}{}\n".format( - chld_cls, - chld_doc_string, - chld_var, - chld_init, - params_init[:-2], - init_pkg, - chld_appn, - params_appn[:-2], - append_pkg, - ) - pb_file.write(packages_str) - pb_file.close() - - init_file_imports.append( - f"from .mf{package_name} import Modflow{package_name.title()}\n" - ) - - if package[0].dfn_type == mfstructure.DfnType.model_name_file: - # build model file - init_vars = build_model_init_vars(options_param_list) - - options_param_list.insert(0, "model_rel_path='.'") - options_param_list.insert(0, "exe_name='mf6'") - options_param_list.insert(0, "version='mf6'") - options_param_list.insert(0, "model_nam_file=None") - options_param_list.insert(0, "modelname='model'") - options_param_list.append("**kwargs,") - init_string_sim = build_init_string( - init_string_sim, options_param_list - ) - sim_name = clean_class_string(package[2]) - class_def_string = "class Modflow{}(mfmodel.MFModel):\n".format( - sim_name.capitalize() - ) - class_def_string = class_def_string.replace("-", "_") - doc_string.add_parameter( - " sim : MFSimulation\n " - "Simulation that this model is a part " - "of. Model is automatically\n " - "added to simulation when it is " - "initialized.", - beginning_of_list=True, - model_parameter=True, - ) - doc_string.description = ( - f"Modflow{sim_name} defines a {sim_name} model" - ) - class_var_string = f" model_type = '{sim_name}'\n" - mparent_init_string = " super().__init__(" - spaces = " " * len(mparent_init_string) - mparent_init_string = ( - "{}simulation, model_type='{}6',\n{}" - "modelname=modelname,\n{}" - "model_nam_file=model_nam_file,\n{}" - "version=version, exe_name=exe_name,\n{}" - "model_rel_path=model_rel_path,\n{}" - "**kwargs," - ")\n".format( - mparent_init_string, - sim_name, - spaces, - spaces, - spaces, - spaces, - spaces, - ) - ) - load_txt, doc_text = build_model_load(sim_name) - package_string = "{}\n{}\n\n\n{}{}\n{}\n{}\n{}{}\n{}\n\n{}".format( - comment_string, - nam_import_string, - class_def_string, - doc_string.get_doc_string(True), - doc_text, - class_var_string, - init_string_sim, - mparent_init_string, - init_vars, - load_txt, - ) - md_file = open( - os.path.join(util_path, "..", "modflow", f"mf{sim_name}.py"), - "w", - newline="\n", - ) - md_file.write(package_string) - md_file.close() - init_file_imports.append( - f"from .mf{sim_name} import Modflow{sim_name.capitalize()}\n" - ) - elif package[0].dfn_type == mfstructure.DfnType.sim_name_file: - # build simulation file - init_vars = build_model_init_vars(options_param_list) - - options_param_list.insert(0, "lazy_io=False") - options_param_list.insert(0, "use_pandas=True") - options_param_list.insert(0, "write_headers=True") - options_param_list.insert(0, "verbosity_level=1") - options_param_list.insert( - 0, "sim_ws: Union[str, os.PathLike] = " "os.curdir" - ) - options_param_list.insert( - 0, "exe_name: Union[str, os.PathLike] " '= "mf6"' - ) - options_param_list.insert(0, "version='mf6'") - options_param_list.insert(0, "sim_name='sim'") - init_string_sim = " def __init__(self" - init_string_sim = build_init_string( - init_string_sim, options_param_list - ) - class_def_string = ( - "class MFSimulation(mfsimbase." "MFSimulationBase):\n" - ) - doc_string.add_parameter( - " sim_name : str\n" " Name of the simulation", - beginning_of_list=True, - model_parameter=True, - ) - doc_string.description = ( - "MFSimulation is used to load, build, and/or save a MODFLOW " - "6 simulation. \n A MFSimulation object must be created " - "before creating any of the MODFLOW 6 \n model objects." - ) - sparent_init_string = " super().__init__(" - spaces = " " * len(sparent_init_string) - sparent_init_string = ( - "{}sim_name=sim_name,\n{}" - "version=version,\n{}" - "exe_name=exe_name,\n{}" - "sim_ws=sim_ws,\n{}" - "verbosity_level=verbosity_level,\n{}" - "write_headers=write_headers,\n{}" - "lazy_io=lazy_io,\n{}" - "use_pandas=use_pandas,\n{}" - ")\n".format( - sparent_init_string, - spaces, - spaces, - spaces, - spaces, - spaces, - spaces, - spaces, - spaces, - ) - ) - sim_import_string = ( - "import os\n" - "from typing import Union\n" - "from .. import mfsimbase" - ) - - load_txt, doc_text = build_sim_load() - package_string = "{}\n{}\n\n\n{}{}\n{}\n{}{}\n{}\n\n{}".format( - comment_string, - sim_import_string, - class_def_string, - doc_string.get_doc_string(False, True), - doc_text, - init_string_sim, - sparent_init_string, - init_vars, - load_txt, - ) - sim_file = open( - os.path.join(util_path, "..", "modflow", "mfsimulation.py"), - "w", - newline="\n", - ) - sim_file.write(package_string) - sim_file.close() - init_file_imports.append( - "from .mfsimulation import MFSimulation\n" - ) - - # Sort the imports - for line in sorted(init_file_imports, key=lambda x: x.split()[3]): - init_file.write(line) - init_file.close() - - -# ------------------------------------------ - - -DFN_PATH = Path(__file__).parent / "data" / "dfn" -TGT_PATH = Path(__file__).parent / "modflow" -SCALAR_TYPES = { - "keyword": bool, - "integer": int, - "double precision": float, - "string": str, -} -NP_SCALAR_TYPES = { - "keyword": np.bool_, - "integer": np.int_, - "double precision": np.float64, - "string": np.str_, -} -TEMPLATE_ENV = Environment(loader=PackageLoader("flopy", "mf6/templates/")) - - -Scalar = Union[bool, int, float, str] -Variable = Dict[str, Scalar] -Variables = Dict[str, Variable] -Metadata = List[str] -Definition = Tuple[Variables, Metadata] -Definitions = Dict[str, Definition] - - -def load_dfn(f) -> Definition: - """ - Load an input definition file. Returns a tuple containing - a dictionary variables and a list of metadata attributes. - """ - meta = list() - vars = dict() - var = dict() - - for line in f: - # remove whitespace/etc from the line - line = line.strip() - - # record flopy metadata attributes but - # skip all other comment lines - if line.startswith("#"): - _, sep, tail = line.partition("flopy") - if sep == "flopy": - meta.append(tail.strip()) - continue - - # if we hit a newline and the parameter dict - # is nonempty, we've reached the end of its - # block of attributes - if not any(line): - if any(var): - vars[var["name"]] = var - var = dict() - continue - - # split the attribute's key and value and - # store it in the parameter dictionary - key, _, value = line.partition(" ") - var[key] = value - - # add the final parameter - if any(var): - vars[var["name"]] = var - - return vars, meta - - -class DefinitionName(NamedTuple): - l: str - r: str - - @property - def title(self) -> str: - l, r = self - if l == "sln": - return r - elif l == "exg": - return r - return f"{l}{r}" - - @property - def parent(self) -> type: - l, r = self - if l in ["exg", "sln"]: - return MFSimulation - if l in ["utl"]: - return MFPackage - return MFModel - - @property - def parent_kind(self) -> str: - return self.parent.__name__.lower().replace("mf", "") - - @property - def description(self) -> str: - l, r = self - title = self.title.title() - parent = self.parent - parent_kind = self.parent_kind - context = ContextType.from_dfn_name(self) - if context == ContextType.Package: - return ( - f"Modflow{title} defines a {r} package " - f"within a {l} {parent_kind}." - ) - elif context == ContextType.Model: - return f"Modflow{title} defines a {l.upper()} model." - elif context == ContextType.Simulation: - return """ - MFSimulation is used to load, build, and/or save a MODFLOW 6 simulation. - A MFSimulation object must be created before creating any of the MODFLOW 6 - model objects.""" - - @property - def target(self) -> str: - l, r = self - - def _name(): - if l == "sim": - if r == "nam": - return "simulation" - else: - return r - elif l == "sln": - return r - elif l == "exg": - return r - return f"{l}{r}" - - return f"mf{_name()}.py" - - -class ContextType(Enum): - Model = "model" - Package = "package" - Simulation = "simulation" - - @property - def base(self): - if self == ContextType.Model: - return MFModel - elif self == ContextType.Package: - return MFPackage - elif self == ContextType.Simulation: - return MFSimulationBase - - @classmethod - def from_dfn_name(cls, dfn_name: DefinitionName) -> "ContextType": - l, r = dfn_name - if l == "sim" and r == "nam": - return ContextType.Simulation - elif r == "nam": - return ContextType.Model - else: - return ContextType.Package - - -def _try_qualify(t) -> str: - """Convert a type to a name suitable for templating.""" - origin = get_origin(t) - args = get_args(t) - if origin is Literal: - args = ['"' + a + '"' for a in args] - return f"{Literal.__name__}[{', '.join(args)}]" - elif origin is Union: - if len(args) == 2 and args[1] is type(None): - return f"{Optional.__name__}[{_try_qualify(args[0])}]" - return ( - f"{Union.__name__}[{', '.join([_try_qualify(a) for a in args])}]" - ) - elif origin is tuple: - return ( - f"{Tuple.__name__}[{', '.join([_try_qualify(a) for a in args])}]" - ) - elif origin is collections.abc.Iterable: - return f"{Iterable.__name__}[{', '.join([_try_qualify(a) for a in args])}]" - elif origin is list: - return f"{List.__name__}[{', '.join([_try_qualify(a) for a in args])}]" - elif origin is np.ndarray: - return f"NDArray[np.{_try_qualify(args[1].__args__[0])}]" - elif origin is np.dtype: - return str(t) - elif isinstance(t, ForwardRef): - return t.__forward_arg__ - elif t is Ellipsis: - return "..." - elif isinstance(t, type): - return t.__qualname__ - return t - - -def _try_enum_value(v: Any) -> Any: - return v.value if isinstance(v, Enum) else v - - -def _try_de_underscore(k: str) -> str: - return k[1:] if k.startswith("_") else k - - -def _render(d: dict) -> dict: - """ - Recursively transform a dictionary for rendering in a template. - """ - - def _render_key(k): - return _try_de_underscore(k) - - def _render_val(v): - return ( - _render(v) - if isinstance(v, dict) - else _try_qualify(_try_enum_value(v)) - ) - - # drop nones, except for default - _d = { - _render_key(k): _render_val(v) - for k, v in d.items() - if (k == "default" or v is not None) - } - - # wrap default in quotation marks if it's a string - default = _d.get("default", None) - if default is not None and isinstance(default, str): - _d["default"] = f'"{default}"' - - return _d - - -class Renderable: - def render(self) -> dict: - """ - Recursively transform the dataclass instance for rendering. - """ - return _render(asdict(self)) - - -@dataclass(frozen=True) -class Subpackage(Renderable): - parent: type - abbreviation: str - data_name: str - param_name: str - record_name: str - - -class ParameterKind(Enum): - Array = "array" - Scalar = "scalar" - Record = "record" - Union = "union" - List = "list" - - @classmethod - def from_type(cls, t: type) -> Optional["ParameterKind"]: - origin = get_origin(t) - try: - if t is np.ndarray or origin is NDArray or origin is ArrayLike: - return ParameterKind.Array - elif origin is collections.abc.Iterable or origin is list: - return ParameterKind.List - elif origin is tuple: - return ParameterKind.Record - elif origin is Union: - return ParameterKind.Union - elif issubclass(t, Scalar): - return ParameterKind.Scalar - return None - except: - pass - - -@dataclass -class Parameter(Renderable): - name: str - _type: type - description: Optional[str] - default: Optional[Any] - children: Optional[Dict[str, "Parameter"]] - subpackage: Optional[Subpackage] - kind: Optional[ParameterKind] - is_choice: bool = False - set_self: bool = False - - def __init__( - self, - name: str, - _type: type, - description: Optional[str] = None, - default: Optional[Any] = None, - parent: Optional["Parameter"] = None, - children: Optional[Dict[str, "Parameter"]] = None, - subpackage: Optional[Subpackage] = None, - kind: Optional[ParameterKind] = None, - is_choice: bool = False, - set_self: bool = False, - ): - self.name = name - self._type = _type - self.description = description - self.default = default - self.parent = parent - self.children = children - self.subpackage = subpackage - # note the general kind of the parameter, - # i.e. the general type theoretic concept. - # this is ofc derivable on demand, but Jinja - # doesn't allow arbitrary expressions, and it - # doesn't seem to have `subclass`-ish filters. - self.kind = kind or ParameterKind.from_type(_type) - self.is_choice = is_choice - self.set_self = set_self - - -Parameters = Dict[str, Parameter] - - -def _try_get_subpkg(metadata: Metadata) -> Optional[Subpackage]: - lines = { - "subpkg": next( - iter(m for m in metadata if m.startswith("subpac")), None - ), - "parent": next( - iter(m for m in metadata if m.startswith("parent")), None - ), - } - - def _subpkg(): - line = lines["subpkg"] - _, record_name, abbreviation, param_name, data_name = line.split() - return { - "abbreviation": abbreviation, - "data_name": data_name, - "param_name": param_name, - "record_name": record_name, - } - - def _parent(): - line = lines["parent"] - _, _, param_type = line.split() - param_type = param_type.lower() - if "simulation" in param_type: - return MFSimulation - elif "model" in param_type: - return MFModel - return MFPackage - - return ( - Subpackage(parent=_parent(), **_subpkg()) - if all(v for v in lines.values()) - else None - ) - - -def get_template_context( - name: DefinitionName, - base: Union[MFSimulationBase, MFModel, MFPackage], - common: Variables, - definition: Definition, - references: Optional[Definitions] = None, -) -> dict: - """ - Convert an input definition to a template rendering context. - - Notes - ----- - Each input definition corresponds to a generated file/class. - - A map of other definitions may be provided, in which case a - parameter in this context may act as kind of "foreign key", - identifying another context (e.g. a subpackage) which this - context "owns". This allows conditionally initializing any - subpackages in `__init__` for input contexts that use them. - """ - - variables, metadata = definition - - def _convert(variable: Variable, wrap: bool = False) -> Parameter: - """ - Transform a variable from its original representation in - an input definition to a parameter for inclusion in type - hints, docstrings, and the `__init__` method's signature. - - This involves expanding nested input hierarchies, mapping - types to roughly equivalent Python primitives/composites, - and other shaping. - - Notes - ----- - If a `default_value` is not provided, keywords are `False` - by default. Everything else is `None` by default. - - If `wrap` is true, scalars will be wrapped as records with - keywords represented as string literals. This is useful for - unions, to distinguish between choices having the same type. - """ - - _name = variable["name"] - _type = variable.get("type", "unknown") - shape = variable.get("shape", None) - shape = None if shape == "" else shape - optional = variable.get("optional", True) - in_record = variable.get("in_record", False) - tagged = variable.get("tagged, False") - description = variable.get("description", "") - children = None - - def _description(description: str) -> str: - """ - Make substitutions from common variables, - remove backslashes from the description. - TODO: insert citations. - """ - description = description.replace("\\", "") - _, replace, tail = description.strip().partition("REPLACE") - if replace: - key, _, replacements = tail.strip().partition(" ") - replacements = eval(replacements) - val = common.get(key, None).get("description", "") - if val is None: - raise ValueError(f"Common variable not found: {key}") - if any(replacements): - return val.replace("{#1}", replacements["{#1}"]) - return val - return description - - def _fields(record_name: str) -> Parameters: - """ - Load a record's fields and recursively convert them. - - Notes - ----- - This function is provided because records - need extra processing; we remove keywords - and 'filein'/'fileout', which are details - of the mf6io format, not of python/flopy. - """ - record = variables[record_name] - field_names = record["type"].split()[1:] - fields: Dict[str, Parameter] = { - n: _convert(field) - for n, field in variables.items() - if n in field_names - } - field_names = list(fields.keys()) - - # if the record represents a file... - if "file" in record_name: - # remove filein/fileout - for term in ["filein", "fileout"]: - if term in field_names: - fields.pop(term) - - # remove leading keyword - keyword = next(iter(fields), None) - if keyword: - fields.pop(keyword) - - # set the type - n = list(fields.keys())[0] - path_field = fields[n] - path_field._type = os.PathLike - fields[n] = path_field - - # if tagged, remove the leading keyword - elif record.get("tagged", False): - keyword = next(iter(fields), None) - if keyword: - fields.pop(keyword) - - return fields - - # now, proceed through the possible input types - # from top (composites) to bottom (primitives): - # - list - # - union - # - record - # - array - # - scalar - - # list input can have records or unions as rows. - # lists which have a consistent record type are - # regular, inconsistent record types irregular. - if _type.startswith("recarray"): - # make sure columns are defined - names = _type.split()[1:] - n_names = len(names) - if n_names < 1: - raise ValueError(f"Missing recarray definition: {_type}") - - # regular tabular/columnar data (1 record type) can be - # defined with a nested record (i.e. explicit) or with - # fields directly inside the recarray (implicit). list - # data for unions/keystrings necessarily comes nested. - - def _is_explicit_record(): - return len(names) == 1 and variables[names[0]][ - "type" - ].startswith("record") - - def _is_implicit_record(): - types = [ - _try_qualify(v["type"]) - for n, v in variables.items() - if n in names - ] - scalar_types = list(SCALAR_TYPES.keys()) - return all(t in scalar_types for t in types) - - if _is_explicit_record(): - record_name = names[0] - record_type = _convert(variables[record_name]) - children = {record_name: record_type} - type_ = Iterable[record_type._type] - elif _is_implicit_record(): - # record implicitly defined, make it on the fly - record_name = _name - record_fields = _fields(record_name) - record_type = Tuple[ - tuple([f._type for f in record_fields.values()]) - ] - record = Parameter( - name=record_name, - _type=record_type, - children=record_fields, - ) - children = {record_name: record} - type_ = Iterable[record_type] - else: - # irregular recarray, rows can be any of several types - children = {n: _convert(variables[n]) for n in names} - type_ = Iterable[ - Union[tuple([c._type for c in children.values()])] - ] - - # basic composite types... - # union (product), children are record choices - elif _type.startswith("keystring"): - names = _type.split()[1:] - children = {n: _convert(variables[n], wrap=True) for n in names} - type_ = Union[tuple([c._type for c in children.values()])] - - # record (sum) type, children are fields - elif _type.startswith("record"): - children = _fields(_name) - if len(children) > 1: - record_type = Tuple[ - tuple([c._type for c in children.values()]) - ] - elif len(children) == 1: - t = list(children.values())[0]._type - # make sure we don't double-wrap tuples - record_type = t if get_origin(t) is tuple else Tuple[(t,)] - # TODO: if record has 1 field, accept value directly? - type_ = record_type - - # are we wrapping a record which is a - # choice within a union? if so, use a - # literal for the keyword as tag e.g. - # `Tuple[Literal[...], T]` - elif wrap: - field_name = _name - field = _convert(variable) - field_type = ( - Literal[field_name] if field._type is bool else field._type - ) - record_type = ( - Tuple[Literal[field_name]] - if field._type is bool - else Tuple[Literal[field_name], field._type] - ) - children = { - field_name: replace(field, _type=field_type, is_choice=True) - } - type_ = record_type - - # at this point, if it has a shape, it's an array.. - # but if it's in a record make it a variadic tuple, - # and if its item type is a string use an iterable. - elif shape is not None: - scalars = list(SCALAR_TYPES.keys()) - if in_record: - if _type not in scalars: - raise TypeError(f"Unsupported repeating type: {_type}") - type_ = Tuple[SCALAR_TYPES[_type], ...] - elif _type in scalars and SCALAR_TYPES[_type] is str: - type_ = Iterable[SCALAR_TYPES[_type]] - else: - if _type not in NP_SCALAR_TYPES.keys(): - raise TypeError(f"Unsupported array type: {_type}") - type_ = NDArray[NP_SCALAR_TYPES[_type]] - - # finally a bog standard scalar - else: - # if it's a keyword, there are two cases we want to convert it to - # a string literal: if it's 1) tagging another variable, or 2) it - # is being wrapped into a record to represent a choice in a union - tag = _type == "keyword" and (tagged or wrap) - type_ = Literal[_name] if tag else SCALAR_TYPES.get(_type, _type) - - # make optional if needed - if optional: - type_ = ( - Optional[type_] - if (type_ is not bool and not in_record and not wrap) - else type_ - ) - - # keywords default to False, everything else to None - default = variable.get("default", False if type_ is bool else None) - - return Parameter( - # if name is a reserved keyword, add a trailing underscore to it - name=f"{_name}_" if _name in kwlist else _name, - # MF6IO type is mapped to a (lenient) Python equivalent - _type=type_, - # default value - default=default, - # do substitutions, expand citations, apply formatting, etc - description=_description(description), - # child parameters - children=children, - # does variable reference another context (e.g. subpkg)? - subpackage=references.get(_name, None) if references else None, - ) - - def _add_exg_params(params: Parameters) -> Parameters: - a = name.r[:3] - b = name.r[:3] - default = f"{a.upper()}6-{b.upper()}6" - return { - "exgtype": Parameter( - **{ - "name": "exgtype", - "_type": str, - "default": default, - "description": "The exchange type.", - "set_self": True, - } - ), - "exgmnamea": Parameter( - **{ - "name": "exgmnamea", - "_type": str, - "description": "The name of the first model in the exchange.", - "set_self": True, - } - ), - "exgmnameb": Parameter( - **{ - "name": "exgmnameb", - "_type": str, - "description": "The name of the second model in the exchange.", - "set_self": True, - } - ), - **params, - } - - def _add_pkg_params(params: Parameters) -> Parameters: - parent_type = name.parent - parent_name = parent_type.__name__.lower().replace("mf", "") - is_subpkg = name[0] in ["utl"] - out = { - parent_name: Parameter( - **{ - "name": parent_name, - "_type": parent_type, - "description": ( - f"{parent_name.title()} that this package is part of. " - f"Package is automatically added to the {parent_name} " - "when it is initialized." - ), - } - ), - "loading_package": Parameter( - **{ - "name": "loading_package", - "_type": bool, - "description": ( - "Do not set this variable. It is intended for debugging " - "and internal processing purposes only." - ), - } - ), - **params, - "filename": Parameter( - **{ - "name": "filename", - "_type": str, - "description": "File name for this package.", - } - ), - "pname": Parameter( - **{ - "name": "pname", - "_type": str, - "description": "Package name for this package.", - } - ), - } - if is_subpkg: - out["parent_file"] = Parameter( - **{ - "name": "parent_file", - "_type": PathLike, - "description": ( - "Parent package file that references this package. Only needed " - "for utility packages (mfutl*). For example, mfutllaktab package " - "must have a mfgwflak package parent_file." - ), - } - ) - return out - - def _add_mdl_params(params: Parameters) -> Parameters: - return { - "simulation": Parameter( - **{ - "name": "simulation", - "_type": MFSimulation, - "description": ( - "Simulation that this model is part of. " - "Model is automatically added to the simulation " - "when it is initialized." - ), - } - ), - "modelname": Parameter( - **{ - "name": "modelname", - "_type": str, - "description": "The name of the model.", - } - ), - "model_nam_file": Parameter( - **{ - "name": "model_nam_file", - "_type": PathLike, - "description": ( - "The relative path to the model name file from model working folder." - ), - } - ), - "version": Parameter( - **{ - "name": "version", - "_type": str, - "description": "The version of modflow", - } - ), - "exe_name": Parameter( - **{ - "name": "exe_name", - "_type": str, - "description": "The executable name.", - } - ), - "model_ws": Parameter( - **{ - "name": "model_ws", - "_type": PathLike, - "description": "The model working folder path.", - } - ), - **params, - } - - def _add_sim_params(params: Parameters) -> Parameters: - sim_params = { - "sim_name": Parameter( - name="sim_name", - _type=str, - default="sim", - description="Name of the simulation.", - ), - "version": Parameter( - name="version", - _type=str, - default="mf6", - ), - "exe_name": Parameter( - name="exe_name", - _type=PathLike, - default="mf6", - ), - "sim_ws": Parameter( - name="sim_ws", - _type=PathLike, - default=os.curdir, - ), - "verbosity_level": Parameter( - name="verbosity_level", - _type=int, - default=1, - ), - "write_headers": Parameter( - name="write_headers", - _type=bool, - default=True, - ), - "use_pandas": Parameter( - name="use_pandas", - _type=bool, - default=True, - ), - "lazy_io": Parameter( - name="lazy_io", - _type=bool, - default=False, - ), - } - return { - **sim_params, - **params, - } - - def _parameters(definition: Definition) -> Parameters: - """ - Convert the input variables to parameters for an input - context class. Context-specific parameters may also be - added depending. - - Notes - ----- - Not all variables become parameters; nested variables - will become components of composite parameters, e.g., - record fields, keystring (union) choices, list items. - """ - - vars, _ = definition - _vars = vars.copy() - params = { - name: _convert(var) - for name, var in _vars.items() - # filter composite components - # since we've already inflated - # parent types in the hierarchy - if not var.get("in_record", False) - } - l, r = name - if l == "sim" and r == "nam": - params = _add_sim_params(params) - elif l == "exg": - params = _add_exg_params(params) - elif r == "nam": - params = _add_mdl_params(params) - else: - params = _add_pkg_params(params) - return {n: p.render() for n, p in params.items()} - - def _dfn(definition: Definition) -> List[Metadata]: - """ - Get a list of the class' original definition attributes, - as an internal/partial reproduction of the DFN contents. - - Notes - ----- - Currently, generated classes have a `.dfn` property that - reproduces the corresponding DFN sans a few attributes. - This represents the DFN in raw form, before adapting to - Python, consolidating nested types, etc. - """ - - vars, meta = definition - - def _to_dfn_fmt(var: Variable) -> List[str]: - exclude = ["longname", "description"] - return [ - " ".join([k, v]) for k, v in var.items() if k not in exclude - ] - - return [["header"] + [attr for attr in meta]] + [ - _to_dfn_fmt(var) for var in vars.values() - ] - - return { - # the input definition's name. - "name": name, - # the corresponding context class name. - "class": name.title, - # the base class the context class inherits from. - "base": _try_qualify(base), - # contexts may be attached to a parent context type. - "parent": { - "name": name.title, - "type": _try_qualify(name.parent), - "kind": name.parent_kind, - }, - # the context's description. header of the docstring. - "description": name.description, - # the context's parameters. inserted into docstring - # and __init__ method, optionally set in __init__. - "parameters": _parameters(definition), - # reproduction of (most of the) original DFN. - "dfn": _dfn(definition), - } - - -def generate_target( - dfn_name: DefinitionName, - out_path: Path, - common: Variables, - definition: Definition, - subpackages: Definitions, - verbose: bool = False, -): - """ - Generate Python source file(s) from the given input definition. - - Notes - ----- - - Model definitions will produce two files / classes, one for the - model itself and one for its corresponding control file package. - """ - - template = TEMPLATE_ENV.get_template("context.jinja") - context_type = ContextType.from_dfn_name(dfn_name) - context_base = context_type.base - target_path = out_path / dfn_name.target - with open(target_path, "w") as f: - context = { - "context": context_type.value, - **get_template_context( - name=dfn_name, - base=context_base, - common=common, - definition=definition, - references=subpackages, - ), - } - source = template.render(**context) - if verbose: - print(f"Generating {target_path}") - f.write(source) - - # bc models are defined in `-nam.dfn` files, - # we wrote the model class file above, also need - # to write the package class file. - if context_type == ContextType.Model: - _dfn_name = DefinitionName(l=dfn_name.l, r="") - target_path = out_path / _dfn_name.target - with open(target_path, "w") as f: - context = { - "context": "package", - **get_template_context( - name=_dfn_name, - base=MFPackage, - common=common, - definition=definition, - references=subpackages, - ), - } - source = template.render(**context) - if verbose: - print(f"Generating {target_path}") - f.write(source) - - -def generate_targets(dfn_path: Path, out_path: Path, verbose: bool = False): - """Generate Python source files from the DFN files in the given location.""" - - # find definition files - dfn_paths = [ - dfn - for dfn in dfn_path.glob("*.dfn") - if dfn.stem not in ["common", "flopy"] - ] - - # try to load common variables - if not any(p.stem == "common" for p in dfn_paths): - warn("No common input definition file...") - with open(dfn_path / "common.dfn") as f: - common, _ = load_dfn(f) - - # load and generate subpackages first so we can - # identify references to them in other contexts - subpkg_paths = [dfn for dfn in dfn_paths if dfn.stem.startswith("utl")] - subpkgs = dict() - for p in subpkg_paths: - l, r = p.stem.split("-") - name = DefinitionName(l, r) - with open(p) as f: - variables, metadata = load_dfn(f) - definition = (variables, metadata) - subpackage = _try_get_subpkg(metadata) - if subpackage: - subpkgs[r] = subpackage - else: - warn(f"{name} lacks subpackage metadata, only: {metadata}") - generate_target( - name, out_path, common, definition, subpkgs, verbose - ) - - # load and generate the rest of the contexts - context_paths = [p for p in dfn_paths if p not in subpkg_paths] - for p in context_paths: - name = DefinitionName(*p.stem.split("-")) - with open(p) as f: - definition = load_dfn(f) - generate_target( - name, out_path, common, definition, subpkgs, verbose - ) - - # format the generated file - run_cmd("ruff", "format", out_path, verbose=verbose) - - -if __name__ == "__main__": - generate_targets(DFN_PATH, TGT_PATH) diff --git a/flopy/mf6/templates/attrs.jinja b/flopy/mf6/templates/attrs.jinja deleted file mode 100644 index d9f383b100..0000000000 --- a/flopy/mf6/templates/attrs.jinja +++ /dev/null @@ -1,16 +0,0 @@ -{# Class attributes for a component class. #} - {% for name, param in parameters.items() %} - {%- if param.kind == "list" or param.kind == "record" %} - {{ param.name }} = ListTemplateGenerator(("{{ component }}6", "{{ subcomponent }}", "{{ param.block }}", "{{ param.name }}")) - {%- elif param.kind == "array" %} - {{ param.name }} = ArrayTemplateGenerator(("{{ component }}6", "{{ subcomponent }}", "{{ param.block }}", "{{ param.name }}")) - {%- endif -%} - {%- endfor %} - {%- if context == "model" %} - model_type = {{ component }} - {%- elif context == "package" %} - package_abbr = "{{ component }}{{ subcomponent }}" - _package_type = "{{ subcomponent }}" - dfn_file_name = "{{ component }}-{{ subcomponent }}.dfn" - dfn = {{ dfn|pprint|indent(10) }} - {% endif -%} \ No newline at end of file diff --git a/flopy/mf6/templates/docstring.jinja b/flopy/mf6/templates/docstring.jinja deleted file mode 100644 index b8ab8a0896..0000000000 --- a/flopy/mf6/templates/docstring.jinja +++ /dev/null @@ -1,23 +0,0 @@ -{# A component class' docstring. #} - {{ description }} - - Parameters - ---------- - {{ parent.name }} : {{ parent.type }} - {{ parent.kind.title() }} that this package is a part of. Package is automatically - added to the parent {{ parent.kind }} when it is initialized. - - loading_package : bool - Do not set this variable. It is intended for debugging and internal - processing purposes only. - {% include "params.jinja" %} - filename : str - File name for this package. - - pname : str - Package name for this package. - - parent_file : PathLike - Parent package file that references this package. Only needed for - utility packages (mfutl*). For example, mfutllaktab package must have - a mfgwflak package parent_file. \ No newline at end of file diff --git a/flopy/mf6/templates/init.jinja b/flopy/mf6/templates/init.jinja deleted file mode 100644 index 0328f08130..0000000000 --- a/flopy/mf6/templates/init.jinja +++ /dev/null @@ -1,67 +0,0 @@ -def __init__( - self, - {%- for name, param in parameters.items() %} - {{ name }}: {{ param.type }} = {{ param.default }}, - {%- endfor %} - **kwargs, - ): -{% if context == "simulation" %} - super().__init__( - {%- for name in parameters.keys() %} - {{ name }}={{ name }}, - {%- endfor %}, - **kwargs - ) - - {%- for name, param in parameters.items() if param.block == "options" %} - self.name_file.{{ name }}.set_data({{ name }}) - self.{{ name }} = self.name_file.{{ name }}, - {%- if param.subpackage is defined %} - self{{ param.subpackage.data_name }} = self._create_package( - "{{ param.subpackage.param_name }}", - self.{{ param.subpackage.data_name }} - ) - {% endif -%} - {% endfor -%} -{% elif context == "model" %} - super().__init__( - model_type="{{ component }}6", - {%- for name in parameters.keys() %} - {{ name }}={{ name }}, - {%- endfor %} - **kwargs, - ) - - {%- for name, param in parameters.items() if param.block == "options" %} - self.name_file.{{ name }}.set_data({{ name }}) - self.{{ name }} = self.name_file.{{ name }}, - {% endfor -%} -{% elif context == "package" %} - super().__init__( - package_type="{{ subcomponent }}", - {%- for name in parameters.keys() %} - {{ name }}={{ name }}, - {%- endfor %} - **kwargs - ) - - {%- for name, param in parameters.items() %} - {%- if param.set_self %} - self.{{ name }} = {{ name }} - {% else %} - self.{{ name }} = self.build_mfdata("{{ name }}", {{ name }}) - {% endif -%} - {%- if param.subpackage is defined %} - self._{{ param.subpackage.abbr }}_package = self.build_child_package( - "{{ param.subpackage.abbrev }}", - {{ param.subpackage.data_name }}, - "{{ param.subpackage.param_name }}", - self.{{ param.subpackage.record_name }} - ) - {% endif -%} - {%- endfor %} - {%- if component == "exg" %} - simulation.register_exchange_file(self) - {% endif -%} - self._init_complete = True -{% endif %} \ No newline at end of file diff --git a/flopy/mf6/templates/model.jinja b/flopy/mf6/templates/model.jinja deleted file mode 100644 index 12350bac3d..0000000000 --- a/flopy/mf6/templates/model.jinja +++ /dev/null @@ -1,141 +0,0 @@ -# autogenerated file, do not modify -from .. import mfmodel -from ..data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenerator - - -class Modflow{{ title }}(mfmodel.MFModel): - """ - Modflow{{ title }} defines a {{ component }} model - - Parameters - ---------- - modelname : string - name of the model - model_nam_file : string - relative path to the model name file from model working folder - version : string - version of modflow - exe_name : string - model executable name - model_ws : string - model working folder path - sim : MFSimulation - Simulation that this model is a part of. Model is automatically - added to simulation when it is initialized. - list : string - * list (string) is name of the listing file to create for this {{ component.upper() }} - model. If not specified, then the name of the list file will be the - basename of the {{ component.upper() }} model name file and the '.lst' extension. For - example, if the {{ component.upper() }} name file is called "my.model.nam" then the list - file will be called "my.model.lst". - print_input : boolean - * print_input (boolean) keyword to indicate that the list of all model - stress package information will be written to the listing file - immediately after it is read. - print_flows : boolean - * print_flows (boolean) keyword to indicate that the list of all model - package flow rates will be printed to the listing file for every - stress period time step in which "BUDGET PRINT" is specified in - Output Control. If there is no Output Control option and - "PRINT_FLOWS" is specified, then flow rates are printed for the last - time step of each stress period. - save_flows : boolean - * save_flows (boolean) keyword to indicate that all model package flow - terms will be written to the file specified with "BUDGET FILEOUT" in - Output Control. - newtonoptions : [under_relaxation] - * under_relaxation (string) keyword that indicates whether the - groundwater head in a cell will be under-relaxed when water levels - fall below the bottom of the model below any given cell. By default, - Newton-Raphson UNDER_RELAXATION is not applied. - packages : [ftype, fname, pname] - * ftype (string) is the file type, which must be one of the following - character values shown in table ref{table:ftype-{{ component }}}. Ftype may be - entered in any combination of uppercase and lowercase. - * fname (string) is the name of the file containing the package input. - The path to the file should be included if the file is not located in - the folder where the program was run. - * pname (string) is the user-defined name for the package. PNAME is - restricted to 16 characters. No spaces are allowed in PNAME. PNAME - character values are read and stored by the program for stress - packages only. These names may be useful for labeling purposes when - multiple stress packages of the same type are located within a single - {{ component.upper() }} Model. If PNAME is specified for a stress package, then PNAME - will be used in the flow budget table in the listing file; it will - also be used for the text entry in the cell-by-cell budget file. - PNAME is case insensitive and is stored in all upper case letters. - - Methods - ------- - load : (simulation : MFSimulationData, model_name : string, - namfile : string, version : string, exe_name : string, - model_ws : string, strict : boolean) : MFSimulation - a class method that loads a model from files - """ - - model_type = "{{ component }}" - - def __init__( - self, - simulation, - modelname="model", - model_nam_file=None, - version="mf6", - exe_name="mf6", - model_rel_path=".", - list=None, - print_input=None, - print_flows=None, - save_flows=None, - newtonoptions=None, - **kwargs, - ): - super().__init__( - simulation, - model_type="{{ component }}6", - modelname=modelname, - model_nam_file=model_nam_file, - version=version, - exe_name=exe_name, - model_rel_path=model_rel_path, - **kwargs, - ) - - self.name_file.list.set_data(list) - self.name_file.print_input.set_data(print_input) - self.name_file.print_flows.set_data(print_flows) - self.name_file.save_flows.set_data(save_flows) - self.name_file.newtonoptions.set_data(newtonoptions) - - self.list = self.name_file.list - self.print_input = self.name_file.print_input - self.print_flows = self.name_file.print_flows - self.save_flows = self.name_file.save_flows - self.newtonoptions = self.name_file.newtonoptions - - @classmethod - def load( - cls, - simulation, - structure, - modelname="NewModel", - model_nam_file="modflowtest.nam", - version="mf6", - exe_name="mf6", - strict=True, - model_rel_path=".", - load_only=None, - ): - return mfmodel.MFModel.load_base( - cls, - simulation, - structure, - modelname, - model_nam_file, - "{{ component }}6", - version, - exe_name, - strict, - model_rel_path, - load_only, - ) diff --git a/flopy/mf6/templates/params.jinja b/flopy/mf6/templates/params.jinja deleted file mode 100644 index 3be82750b4..0000000000 --- a/flopy/mf6/templates/params.jinja +++ /dev/null @@ -1,9 +0,0 @@ -{% for param in parameters.values() recursive %} - {% if loop.depth > 1 %}* {% endif %}{{ param.name }} : {{ param.type }} -{%- if param.description is defined and not param.is_choice %} -{{ param.description|wordwrap|indent(4 + (loop.depth * 4), first=True) }} -{%- endif %} -{%- if param.children is defined -%} -{{ loop(param.children.values())|indent(4) }} -{%- endif %} -{% endfor %} \ No newline at end of file diff --git a/flopy/mf6/templates/simulation.jinja b/flopy/mf6/templates/simulation.jinja deleted file mode 100644 index a85ff9a657..0000000000 --- a/flopy/mf6/templates/simulation.jinja +++ /dev/null @@ -1,99 +0,0 @@ -# autogenerated file, do not modify -import os -from typing import Union - -from .. import mfsimbase - - -class MFSimulation(mfsimbase.MFSimulationBase): - """ - MFSimulation is used to load, build, and/or save a MODFLOW 6 simulation. - A MFSimulation object must be created before creating any of the MODFLOW 6 - model objects. - - Parameters - ---------- - sim_name : str - Name of the simulation - {% include "params.jinja" %} - - Methods - ------- - load : (sim_name : str, version : string, - exe_name : str or PathLike, sim_ws : str or PathLike, strict : bool, - verbosity_level : int, load_only : list, verify_data : bool, - write_headers : bool, lazy_io : bool, use_pandas : bool, - ) : MFSimulation - a class method that loads a simulation from files - """ - - def __init__( - self, - sim_name="sim", - version="mf6", - exe_name: Union[str, os.PathLike] = "mf6", - sim_ws: Union[str, os.PathLike] = os.curdir, - verbosity_level=1, - write_headers=True, - use_pandas=True, - lazy_io=False, - continue_=None, - nocheck=None, - memory_print_option=None, - maxerrors=None, - print_input=None, - hpc_data=None, - ): - super().__init__( - sim_name=sim_name, - version=version, - exe_name=exe_name, - sim_ws=sim_ws, - verbosity_level=verbosity_level, - write_headers=write_headers, - lazy_io=lazy_io, - use_pandas=use_pandas, - ) - - self.name_file.continue_.set_data(continue_) - self.name_file.nocheck.set_data(nocheck) - self.name_file.memory_print_option.set_data(memory_print_option) - self.name_file.maxerrors.set_data(maxerrors) - self.name_file.print_input.set_data(print_input) - - self.continue_ = self.name_file.continue_ - self.nocheck = self.name_file.nocheck - self.memory_print_option = self.name_file.memory_print_option - self.maxerrors = self.name_file.maxerrors - self.print_input = self.name_file.print_input - self.hpc_data = self._create_package("hpc", hpc_data) - - @classmethod - def load( - cls, - sim_name="modflowsim", - version="mf6", - exe_name: Union[str, os.PathLike] = "mf6", - sim_ws: Union[str, os.PathLike] = os.curdir, - strict=True, - verbosity_level=1, - load_only=None, - verify_data=False, - write_headers=True, - lazy_io=False, - use_pandas=True, - ): - return mfsimbase.MFSimulationBase.load( - cls, - sim_name, - version, - exe_name, - sim_ws, - strict, - verbosity_level, - load_only, - verify_data, - write_headers, - lazy_io, - use_pandas, - ) diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py new file mode 100644 index 0000000000..a144f49977 --- /dev/null +++ b/flopy/mf6/utils/createpackages.py @@ -0,0 +1,1457 @@ +""" +createpackages.py is a utility script that reads in the file definition +metadata in the .dfn files and creates the package classes in the modflow +folder. Run this script any time changes are made to the .dfn files. + +To create a new package that is part of an existing model, first create a new +dfn file for the package in the mf6/data/dfn folder. +1) Follow the file naming convention -.dfn. +2) Run this script (createpackages.py), and check in your new dfn file, and + the package class and updated __init__.py that createpackages.py created. + +A subpackage is a package referenced by another package (vs being referenced +in the name file). The tas, ts, and obs packages are examples of subpackages. +There are a few additional steps required when creating a subpackage +definition file. First, verify that the parent package's dfn file has a file +record for the subpackage to the option block. For example, for the time +series package the file record definition starts with: + + block options + name ts_filerecord + type record ts6 filein ts6_filename + +Verify that the same naming convention is followed as the example above, +specifically: + + name _filerecord + record 6 filein 6_filename + +Next, create the child package definition file in the mf6/data/dfn folder +following the naming convention above. + +When your child package is ready for release follow the same procedure as +other packages along with these a few additional steps required for +subpackages. + +At the top of the child dfn file add two lines describing how the parent and +child packages are related. The first line determines how the subpackage is +linked to the package: + +# flopy subpackage + + +* Parent record is the MF6 record name of the filerecord in parent package + that references the child packages file name +* Abbreviation is the short abbreviation of the new subclass +* Child data is the name of the child class data that can be passed in as + parameter to the parent class. Passing in this parameter to the parent class + automatically creates the child class with the data provided. +* Data name is the parent class parameter name that automatically creates the + child class with the data provided. + +The example below is the first line from the ts subpackage dfn: + +# flopy subpackage ts_filerecord ts timeseries timeseries + +The second line determines the variable name of the subpackage's parent and +the type of parent (the parent package's object oriented parent): + +# flopy parent_name_type + + +An example below is the second line in the ts subpackage dfn: + +# flopy parent_name_type parent_package MFPackage + +There are three possible types (or combination of them) that can be used for +"parent package type", MFPackage, MFModel, and MFSimulation. If a package +supports multiple types of parents (for example, it can be either in the model +namefile or in a package, like the obs package), include all the types +supported, separating each type with a / (MFPackage/MFModel). + +To create a new type of model choose a unique three letter model abbreviation +("gwf", "gwt", ...). Create a name file dfn with the naming convention +-nam.dfn. The name file must have only an options and packages +block (see gwf-nam.dfn as an example). Create a new dfn file for each of the +packages in your new model, following the naming convention described above. + +When your model is ready for release make sure all the dfn files are in the +flopy/mf6/data/dfn folder, run createpackages.py, and check in your new dfn +files, the package classes, and updated init.py that createpackages.py created. + +""" + +import collections +import os +from collections import UserDict +from dataclasses import asdict, dataclass, replace +from enum import Enum +from keyword import kwlist +from os import PathLike +from pathlib import Path +from typing import ( + Any, + Dict, + ForwardRef, + Iterable, + Iterator, + List, + Literal, + NamedTuple, + Optional, + Tuple, + Union, + get_args, + get_origin, +) +from warnings import warn + +import numpy as np +from jinja2 import Environment, PackageLoader +from modflow_devtools.misc import run_cmd +from numpy.typing import ArrayLike, NDArray + +from flopy.mf6 import MFSimulation + +# keep below as absolute imports +from flopy.mf6.mfmodel import MFModel +from flopy.mf6.mfpackage import MFPackage +from flopy.mf6.mfsimbase import MFSimulationBase + + +def _try_get_type_name(t) -> str: + """Convert a type to a name suitable for templating.""" + origin = get_origin(t) + args = get_args(t) + if origin is Literal: + args = ['"' + a + '"' for a in args] + return f"{Literal.__name__}[{', '.join(args)}]" + elif origin is Union: + if len(args) == 2 and args[1] is type(None): + return f"{Optional.__name__}[{_try_get_type_name(args[0])}]" + return f"{Union.__name__}[{', '.join([_try_get_type_name(a) for a in args])}]" + elif origin is tuple: + return f"{Tuple.__name__}[{', '.join([_try_get_type_name(a) for a in args])}]" + elif origin is collections.abc.Iterable: + return f"{Iterable.__name__}[{', '.join([_try_get_type_name(a) for a in args])}]" + elif origin is list: + return f"{List.__name__}[{', '.join([_try_get_type_name(a) for a in args])}]" + elif origin is np.ndarray: + return f"NDArray[np.{_try_get_type_name(args[1].__args__[0])}]" + elif origin is np.dtype: + return str(t) + elif isinstance(t, ForwardRef): + return t.__forward_arg__ + elif t is Ellipsis: + return "..." + elif isinstance(t, type): + return t.__qualname__ + return t + + +def _try_get_enum_value(v: Any) -> Any: + return v.value if isinstance(v, Enum) else v + + +def renderable( + maybe_cls=None, + *, + wrap_str: Optional[List[str]] = None, + keep_none: Optional[List[str]] = None, +): + """ + An object meant to be passed into a template + as a "rendered" dictionary, where "rendering" + means transforming key/value pairs to a form + more convenient for use within the template. + + The object *must* be a dataclass. + + Notes + ----- + Jinja supports attribute- and dictionary- + based access but no arbitrary expressions, + and only a limited set of custom filters. + This can make it awkward to express some + conditionals. Conversely, Python classes + have some limitations (like no attribute + name collisions with reserved keywords). + So we convert the dataclasses we want to + pass to `template.render(...)` to dicts, + with a few touchups. + + These include: + - removing key/value pairs whose value is None + - converting types to their suitably qualified names + - removing leading underscores (due to reserved keyword + collisions) from attribute names + - quoting strings forming the RHS of an assignment or + argument passing expression + + """ + + wrap_str = wrap_str or list() + keep_none = keep_none or list() + + def __renderable(cls): + def _render(d: dict) -> dict: + def _render_val(v): + return _try_get_type_name(_try_get_enum_value(v)) + + # drop nones except for any explicitly kept + _d = { + k: _render_val(v) + for k, v in d.items() + if (k in keep_none or v is not None) + } + + # wrap string values where requested + if wrap_str: + for k in wrap_str: + v = _d.get(k, None) + if v is not None and isinstance(v, str): + _d[k] = f'"{v}"' + + return _d + + def render(self) -> dict: + """ + Recursively render the dataclass instance. + """ + return _render( + asdict(self, dict_factory=lambda d: _render(dict(d))) + ) + + setattr(cls, "_render", _render) + setattr(cls, "render", render) + return cls + + # first arg value depends on the decorator usage: + # class if `@renderable`, `None` if `@renderable()`. + # referenced from https://github.com/python-attrs/attrs/blob/a59c5d7292228dfec5480388b5f6a14ecdf0626c/src/attr/_next_gen.py#L405C4-L406C65 + return __renderable if maybe_cls is None else __renderable(maybe_cls) + + +class ContextName(NamedTuple): + """ + Uniquely identifies an input context by its name, which + consists of a <= 3-letter left term and optional right + term also of <= 3 letters. + + Notes + ----- + A single `DefinitionName` may be associated with one or + more `ContextName`s. For instance, a model DFN file will + produce both a NAM package class and also a model class. + + From the `ContextName` several other things are derived, + including: + + - the name of the source file to write + - the name of the input context class + - a description for the context class + - an optional base class for the context to inherit from + - an optional parent class which can own context instances + + """ + + l: str + r: Optional[str] + + @property + def title(self) -> str: + """ + The input context's unique title. This is not + identical to `f"{l}{r}` in some cases, but it + remains unique. The title is substituted into + the file name and class name. + """ + + l, r = self + if self == ("sim", "nam"): + return "simulation" + if r is None: + return l + if l in ["sln", "exg"]: + return r + return f"{l}{r}" + + @property + def base(self) -> Optional[type]: + """A base class from which the input context should inherit.""" + l, r = self + if self == ("sim", "nam"): + return MFSimulation + if r is None: + return MFModel + return MFPackage + + @property + def parent(self) -> Optional[type]: + """A parent context which owns and manages the input context.""" + l, r = self + if (l, r) == ("sim", "nam"): + return None + if l in ["sim", "exg", "sln"]: + return MFSimulation + if l == "utl": + return MFPackage + return MFModel + + @property + def target(self) -> str: + """The source file name to generate.""" + return f"mf{self.title}.py" + + @property + def description(self) -> str: + """A description of the input context.""" + l, r = self + title = self.title.title() + if self.base == MFPackage: + return ( + f"Modflow{title} defines a {r} package within a {l} " + f"{self.base.__name__.lower().replace('mf', '')}." + ) + elif self.base == MFModel: + return f"Modflow{title} defines a {l.upper()} model." + elif self.base == MFSimulation: + return """ + MFSimulation is used to load, build, and/or save a MODFLOW 6 simulation. + A MFSimulation object must be created before creating any of the MODFLOW 6 + model objects.""" + + +class DfnName(NamedTuple): + """ + Uniquely identifies an input definition by its name, which + consists of a <= 3-letter left term and an optional right + term, also <= 3 letters. + """ + + l: str + r: str + + @property + def contexts(self) -> List[ContextName]: + if self.l != "sim" and self.r == "nam": + return [ + ContextName(*self), # nam pkg + ContextName(self.l, None), # model + ] + return [ContextName(*self)] + + +Metadata = List[str] + + +@dataclass +class Dfn(UserDict): + """ + An MF6 input definition. Contains variables and metadata. + """ + + name: Optional[DfnName] + metadata: Optional[Metadata] + + def __init__( + self, + variables: Dict[str, Dict[str, str]], + name: Optional[DfnName] = None, + metadata: Optional[Metadata] = None, + ): + super().__init__(variables) + self.name = name + self.metadata = metadata + + +Dfns = Dict[str, Dfn] + + +def load_dfn(f, name: Optional[DfnName] = None) -> Dfn: + """ + Load an input definition file. Returns a tuple containing + a dictionary of variable specifications as well as a list + of metadata attributes. + """ + meta = None + vars_ = dict() + var = dict() + + for line in f: + # remove whitespace/etc from the line + line = line.strip() + + # record context name and flopy metadata + # attributes, skip all other comment lines + if line.startswith("#"): + _, sep, tail = line.partition("flopy") + if sep == "flopy": + if meta is None: + meta = list() + meta.append(tail.strip()) + continue + + # if we hit a newline and the parameter dict + # is nonempty, we've reached the end of its + # block of attributes + if not any(line): + if any(var): + vars_[var["name"]] = var + var = dict() + continue + + # split the attribute's key and value and + # store it in the parameter dictionary + key, _, value = line.partition(" ") + var[key] = value + + # add the final parameter + if any(var): + vars_[var["name"]] = var + + return Dfn(variables=vars_, name=name, metadata=meta) + + +@dataclass +class Subpkg: + """ + A foreign-key-like reference between a file input variable + and a subpackage definition. This allows an input context + to reference a subpackage by including a variable with an + appropriate name. + + Parameters + ---------- + key : str + The name of the file input variable identifying the + referenced subpackage. + val : str + The name of the variable containing subpackage data + in the referenced subpackage. + abbr : str + An abbreviation of the subpackage's name. + param : str + The subpackage parameter name. TODO: explain + parents : List[type] + The subpackage's supported parent types. + """ + + key: str + val: str + abbr: str + param: str + parents: List[type] + + @classmethod + def from_dfn(cls, dfn: Dfn) -> Optional["Subpkg"]: + if not dfn.metadata: + return None + + lines = { + "subpkg": next( + iter(m for m in dfn.metadata if m.startswith("subpac")), None + ), + "parent": next( + iter(m for m in dfn.metadata if m.startswith("parent")), None + ), + } + + def _subpkg(): + line = lines["subpkg"] + _, key, abbr, param, val = line.split() + return { + "key": key, + "val": val, + "abbr": abbr, + "param": param, + } + + def _parents(): + line = lines["parent"] + _, _, _type = line.split() + _type = _type.lower() + + def _parent(t): + if "simulation" in t: + return MFSimulation + elif "model" in t: + return MFModel + return MFPackage + + return [_parent(pt) for pt in _type.split("/")] + + return ( + cls(**_subpkg(), parents=_parents()) + if all(v for v in lines.values()) + else None + ) + + +Subpkgs = Dict[str, Subpkg] + + +class VarKind(Enum): + """ + An input variable's kind. This is an enumeration + of the general shapes of data MODFLOW 6 accepts. + """ + + Array = "array" + Scalar = "scalar" + Record = "record" + Union = "union" + List = "list" + + @classmethod + def from_type(cls, t: type) -> Optional["VarKind"]: + origin = get_origin(t) + if t is np.ndarray or origin is NDArray or origin is ArrayLike: + return VarKind.Array + elif origin is collections.abc.Iterable or origin is list: + return VarKind.List + elif origin is tuple: + return VarKind.Record + elif origin is Union: + return VarKind.Union + try: + if issubclass(t, (bool, int, float, str)): + return VarKind.Scalar + except: + pass + return None + + +@dataclass +class Var: + """A variable in a MODFLOW 6 input context.""" + + name: str + _type: Optional[type] + block: Optional[str] + description: Optional[str] + default: Optional[Any] + children: Optional[Dict[str, "Var"]] + meta: Optional[List[str]] + subpkg: Optional[Subpkg] + kind: Optional[VarKind] + is_choice: bool = False + init_param: bool = True + init_assign: bool = False + init_build: bool = False + init_super: bool = False + + def __init__( + self, + name: str, + _type: Optional[type] = None, + block: Optional[str] = None, + description: Optional[str] = None, + default: Optional[Any] = None, + parent: Optional["Var"] = None, + children: Optional["Vars"] = None, + meta: Optional[Metadata] = None, + subpkg: Optional[Subpkg] = None, + kind: Optional[VarKind] = None, + is_choice: bool = False, + init_param: bool = True, + init_assign: bool = False, + init_build: bool = False, + init_super: bool = False, + ): + self.name = name + self._type = _type + self.block = block + self.description = description + self.default = default + self.parent = parent + self.children = children + self.meta = meta + self.subpkg = subpkg + # the variable's general kind. + # this is ofc derivable on demand but Jinja + # doesn't allow arbitrary expressions, and it + # doesn't seem to have `subclass`-ish filters. + self.kind = kind or VarKind.from_type(_type) + # similarly, whether the variable is a choice + # in a union. this is derivable from .parent, + # but awkward to do in Jinja, so use a flag. + self.is_choice = is_choice + # whether the var is an init method parameter + self.init_param = init_param + # whether to assign arguments to self in the + # init method body. if this is false, assume + # the template has conditionals for any more + # involved initialization needs. + self.init_assign = init_assign + # whether to call `build_mfdata()` to build + # the parameter. + self.init_build = init_build + # whether to pass arg to super().__init__() + self.init_super = init_super + + +Vars = Dict[str, Var] + + +@renderable(wrap_str=["default"], keep_none=["block", "default"]) +@dataclass +class Context: + """ + An input context. Each of these is specified by a definition file + and becomes a generated class. A definition file may specify more + than one input context (e.g. model DFNs yield a model class and a + package class). + + A context class minimally consists of a name, a map of variables, + and a list of metadata. + + The context class may inherit from a base class, and may specify + a parent context within which it can be created (the parent then + becomes the first `__init__` method parameter). + + """ + + name: ContextName + base: Optional[type] + parent: Optional[type] + description: Optional[str] + variables: Vars + metadata: Metadata + + +_SCALAR_TYPES = { + "keyword": bool, + "integer": int, + "double precision": float, + "string": str, +} +_NP_SCALAR_TYPES = { + "keyword": np.bool_, + "integer": np.int_, + "double precision": np.float64, + "string": np.str_, +} + + +def make_context( + name: ContextName, + dfn: Dfn, + common: Optional[Dfn] = None, + subpkgs: Optional[Subpkgs] = None, +) -> Context: + """ + Convert an MF6 input definition to a structured descriptor + of an input context class to create with a Jinja template. + + Notes + ----- + Each input definition corresponds to a generated Python + source file. A definition may produce one or more input + context classes. + + A map of other definitions may be provided, in which case a + parameter in this context may act as kind of "foreign key", + identifying another context as a subpackage which this one + is related to. + """ + + subpkgs = subpkgs or dict() + common = common or dict() + + def _convert( + var: Dict[str, str], + wrap: bool = False, + ) -> Var: + """ + Transform a variable from its original representation in + an input definition to a specification suitable for type + hints, docstrings, an `__init__` method's signature, etc. + + This involves expanding nested input hierarchies, mapping + types to roughly equivalent Python primitives/composites, + and other shaping. + + Notes + ----- + If a `default_value` is not provided, keywords are `False` + by default. Everything else is `None` by default. + + If `wrap` is true, scalars will be wrapped as records with + keywords represented as string literals. This is useful for + unions, to distinguish between choices having the same type. + + A map of subpackage references `refs` may be provided, in + which case any variable whose name is a key for any given + subpackage will detect and store the subpackage reference. + """ + + # var attributes to be converted + _name = var["name"] + _type = var.get("type", "unknown") + block = var.get("block", None) + shape = var.get("shape", None) + shape = None if shape == "" else shape + optional = var.get("optional", True) + in_record = var.get("in_record", False) + tagged = var.get("tagged, False") + description = var.get("description", "") + children = None + + def _description(descr: str) -> str: + """ + Make substitutions from common variable definitions, + remove backslashes, generate/insert citations, etc. + TODO: insert citations. + """ + descr = descr.replace("\\", "") + _, replace, tail = descr.strip().partition("REPLACE") + if replace: + key, _, replacements = tail.strip().partition(" ") + replacements = eval(replacements) + common_var = common.get(key, None) + if common_var is None: + raise ValueError(f"Common variable not found: {key}") + descr = common_var.get("description", "") + if any(replacements): + return descr.replace("\\", "").replace( + "{#1}", replacements["{#1}"] + ) + return descr + return descr + + def _fields(record_name: str) -> Vars: + """ + Load a record's fields and recursively convert them. + + Notes + ----- + This function is provided because records + need extra processing; we remove keywords + and 'filein'/'fileout', which are details + of the mf6io format, not of python/flopy. + """ + record = dfn[record_name] + field_names = record["type"].split()[1:] + fields: Dict[str, Var] = { + n: _convert(field, wrap=False) + for n, field in dfn.items() + if n in field_names + } + field_names = list(fields.keys()) + + # if the record represents a file... + if "file" in record_name: + # remove filein/fileout + for term in ["filein", "fileout"]: + if term in field_names: + fields.pop(term) + + # remove leading keyword + keyword = next(iter(fields), None) + if keyword: + fields.pop(keyword) + + # set the type + n = list(fields.keys())[0] + path_field = fields[n] + path_field._type = Union[str, os.PathLike] + fields[n] = path_field + + # if tagged, remove the leading keyword + elif record.get("tagged", False): + keyword = next(iter(fields), None) + if keyword: + fields.pop(keyword) + + return fields + + # go through all the possible input types + # from top (composite) to bottom (scalar): + # + # - list + # - union + # - record + # - array + # - scalar + # + # list input can have records or unions as rows. + # lists which have a consistent record type are + # regular, inconsistent record types irregular. + if _type.startswith("recarray"): + # make sure columns are defined + names = _type.split()[1:] + n_names = len(names) + if n_names < 1: + raise ValueError(f"Missing recarray definition: {_type}") + + # regular tabular/columnar data (1 record type) can be + # defined with a nested record (i.e. explicit) or with + # fields directly inside the recarray (implicit). list + # data for unions/keystrings necessarily comes nested. + + is_explicit_record = len(names) == 1 and dfn[names[0]][ + "type" + ].startswith("record") + + def _is_implicit_record(): + types = [ + _try_get_type_name(v["type"]) + for n, v in dfn.items() + if n in names + ] + scalar_types = list(_SCALAR_TYPES.keys()) + return all(t in scalar_types for t in types) + + if is_explicit_record: + record_name = names[0] + record_spec = dfn[record_name] + record_type = _convert(record_spec, wrap=False) + children = {record_name: record_type} + type_ = Iterable[record_type._type] + elif _is_implicit_record(): + record_name = _name + record_fields = _fields(record_name) + record_type = Tuple[ + tuple([f._type for f in record_fields.values()]) + ] + record = Var( + name=record_name, + _type=record_type, + block=block, + children=record_fields, + ) + children = {record_name: record} + type_ = Iterable[record_type] + else: + # irregular recarray, rows can be any of several types + children = {n: _convert(dfn[n], wrap=False) for n in names} + type_ = Iterable[ + Union[tuple([c._type for c in children.values()])] + ] + + # basic composite types... + # union (product), children are record choices + elif _type.startswith("keystring"): + names = _type.split()[1:] + children = {n: _convert(dfn[n], wrap=True) for n in names} + type_ = Union[tuple([c._type for c in children.values()])] + + # record (sum) type, children are fields + elif _type.startswith("record"): + children = _fields(_name) + if len(children) > 1: + record_type = Tuple[ + tuple([c._type for c in children.values()]) + ] + elif len(children) == 1: + t = list(children.values())[0]._type + # make sure we don't double-wrap tuples + record_type = t if get_origin(t) is tuple else Tuple[(t,)] + # TODO: if record has 1 field, accept value directly? + type_ = record_type + + # are we wrapping a record which is a + # choice within a union? if so, use a + # literal for the keyword as tag e.g. + # `Tuple[Literal[...], T]` + elif wrap: + field_name = _name + field = _convert(var, wrap=False) + field_type = ( + Literal[field_name] if field._type is bool else field._type + ) + record_type = ( + Tuple[Literal[field_name]] + if field._type is bool + else Tuple[Literal[field_name], field._type] + ) + children = { + field_name: replace(field, _type=field_type, is_choice=True) + } + type_ = record_type + + # at this point, if it has a shape, it's an array.. + # but if it's in a record make it a variadic tuple, + # and if its item type is a string use an iterable. + elif shape is not None: + scalars = list(_SCALAR_TYPES.keys()) + if in_record: + if _type not in scalars: + raise TypeError(f"Unsupported repeating type: {_type}") + type_ = Tuple[_SCALAR_TYPES[_type], ...] + elif _type in scalars and _SCALAR_TYPES[_type] is str: + type_ = Iterable[_SCALAR_TYPES[_type]] + else: + if _type not in _NP_SCALAR_TYPES.keys(): + raise TypeError(f"Unsupported array type: {_type}") + type_ = NDArray[_NP_SCALAR_TYPES[_type]] + + # finally a bog standard scalar + else: + # if it's a keyword, there are two cases we want to convert it to + # a string literal: if it's 1) tagging another variable, or 2) it + # is being wrapped into a record to represent a choice in a union + tag = _type == "keyword" and (tagged or wrap) + type_ = Literal[_name] if tag else _SCALAR_TYPES.get(_type, _type) + + # make optional if needed + if optional: + type_ = ( + Optional[type_] + if (type_ is not bool and not in_record and not wrap) + else type_ + ) + + # format the variable description + description = _description(description) + + # keywords default to False, everything else to None + default = var.get("default", False if type_ is bool else None) + + # if name is a reserved keyword, add a trailing underscore to it + name_ = f"{_name}_" if _name in kwlist else _name + + # create var + var_ = Var( + name=name_, + _type=type_, + block=block, + description=description, + default=default, + children=children, + init_param=True, + init_build=True, + ) + + # check if the variable references a subpackage + subpkg = subpkgs.get(_name, None) + if subpkg: + var_.subpkg = subpkg + + return var_ + + def _variables() -> Vars: + """ + Convert the input variables to parameters for an input + context class. Context-specific parameters may also be + added depending. + + Notes + ----- + Not all variables become parameters; nested variables + will become components of composite parameters, e.g., + record fields, keystring (union) choices, list items. + """ + + vars_ = dfn.copy() + vars_ = { + name: _convert(var, wrap=False) + for name, var in vars_.items() + # filter composite components + # since we've already inflated + # their parents in the hierarchy + if not var.get("in_record", False) + } + # set the name since we may have altered + # it when creating the variable (e.g. to + # avoid name/reserved keyword collisions. + vars_ = {v.name: v for v in vars_.values()} + + def _add_exg_vars(_vars: Vars) -> Vars: + """ + Add initializer parameters for an exchange context. + Exchanges need different parameters than a typical + package. + """ + a = name.r[:3] + b = name.r[:3] + default = f"{a.upper()}6-{b.upper()}6" + vars_ = { + "simulation": Var( + name="simulation", + _type=MFSimulation, + description=( + "Simulation that this package is a part of. " + "Package is automatically added to simulation " + "when it is initialized." + ), + init_param=True, + init_assign=False, + init_build=False, + init_super=False, + ), + "loading_package": Var( + name="loading_package", + _type=bool, + description=( + "Do not set this parameter. It is intended for " + "debugging and internal processing purposes only." + ), + init_param=True, + init_assign=False, + init_build=False, + init_super=True, + ), + "exgtype": Var( + name="exgtype", + _type=str, + default=default, + description="The exchange type.", + init_param=True, + init_assign=True, + init_build=False, + init_super=False, + ), + "exgmnamea": Var( + name="exgmnamea", + _type=str, + description="The name of the first model in the exchange.", + init_param=True, + init_assign=True, + init_super=False, + ), + "exgmnameb": Var( + name="exgmnameb", + _type=str, + description="The name of the second model in the exchange.", + init_param=True, + init_assign=True, + init_build=False, + init_super=False, + ), + **_vars, + "filename": Var( + name="filename", + _type=Union[str, PathLike], + description="File name for this package.", + init_param=True, + init_assign=False, + init_build=False, + init_super=True, + ), + "pname": Var( + name="pname", + _type=str, + description="Package name for this package.", + init_param=True, + init_assign=False, + init_build=False, + init_super=True, + ), + } + + # if a reference map is provided, + # find any variables referring to + # subpackages, and attach another + # "value" variable for them all.. + # allows passing data directly to + # `__init__` instead of a path to + # load the subpackage from. maybe + # impossible if the data variable + # doesn't appear in the reference + # definition, though. + if subpkgs: + for k, subpkg in subpkgs.items(): + key = vars_.get(k, None) + if not key: + continue + vars_[subpkg.val] = Var( + name=subpkg.val, + init_param=True, + init_assign=False, + init_super=False, + init_build=False, + ) + + return vars_ + + def _add_pkg_vars(_vars: Vars) -> Vars: + """Add variables for a package context.""" + parent_type = name.parent + parent_name = parent_type.__name__.lower().replace("mf", "") + vars_ = { + parent_name: Var( + name=parent_name, + _type=parent_type, + description=( + f"{parent_name.title()} that this package is part of. " + f"Package is automatically added to the {parent_name} " + "when it is initialized." + ), + init_param=True, + init_assign=False, + init_super=False, + ), + "loading_package": Var( + name="loading_package", + _type=bool, + description=( + "Do not set this variable. It is intended for debugging " + "and internal processing purposes only." + ), + init_param=True, + init_assign=False, + init_super=True, + ), + **_vars, + "filename": Var( + name="filename", + _type=str, + description="File name for this package.", + init_param=True, + init_assign=False, + init_super=True, + ), + "pname": Var( + name="pname", + _type=str, + description="Package name for this package.", + init_param=True, + init_assign=False, + init_super=True, + ), + } + + # if context is a subpackage add + # a `parent_file` variable which + # is the path to the subpackage's + # parent context + subpkg = Subpkg.from_dfn(dfn) + if subpkg: + vars_["parent_file"] = Var( + name="parent_file", + _type=Union[str, PathLike], + description=( + "Parent package file that references this package. Only needed " + "for utility packages (mfutl*). For example, mfutllaktab package " + "must have a mfgwflak package parent_file." + ), + init_param=True, + init_assign=False, + ) + + # if a reference map is provided, + # find any variables referring to + # subpackages, and attach another + # "value" variable for them all.. + # allows passing data directly to + # `__init__` instead of a path to + # load the subpackage from. maybe + # impossible if the data variable + # doesn't appear in the reference + # definition, though. + if subpkgs: + for k, subpkg in subpkgs.items(): + key = vars_.get(k, None) + if not key: + continue + vars_[subpkg.val] = Var( + name=subpkg.val, + init_param=True, + init_assign=False, + init_super=False, + init_build=False, + ) + + return vars_ + + def _add_mdl_vars(_vars: Vars) -> Vars: + """Add variables for a model context.""" + vars_ = _vars.copy() + packages = _vars.get("packages", None) + if packages: + packages.init_param = False + vars_["packages"] = packages + return { + "simulation": Var( + name="simulation", + _type=MFSimulation, + description=( + "Simulation that this model is part of. " + "Model is automatically added to the simulation " + "when it is initialized." + ), + init_param=True, + init_assign=False, + init_super=True, + ), + "modelname": Var( + name="modelname", + _type=str, + description="The name of the model.", + default="model", + init_param=True, + init_assign=False, + init_super=True, + ), + "model_nam_file": Var( + name="model_nam_file", + _type=Optional[Union[str, PathLike]], + description=( + "The relative path to the model name file from model working folder." + ), + init_param=True, + init_assign=False, + init_super=True, + ), + "version": Var( + name="version", + _type=str, + description="The version of modflow", + default="mf6", + init_param=True, + init_assign=False, + init_super=True, + ), + "exe_name": Var( + name="exe_name", + _type=str, + description="The executable name.", + default="mf6", + init_param=True, + init_assign=False, + init_super=True, + ), + "model_rel_path": Var( + name="model_ws", + _type=Union[str, PathLike], + description="The model working folder path.", + default=os.curdir, + init_param=True, + init_assign=False, + init_super=True, + ), + **vars_, + } + + def _add_sim_params(_vars: Vars) -> Vars: + """Add variables for a simulation context.""" + vars_ = _vars.copy() + skip_init = [ + "tdis6", + "models", + "exchanges", + "mxiter", + "solutiongroup", + ] + for k in skip_init: + var = vars_.get(k, None) + if var: + var.init_param = False + vars_[k] = var + return { + "sim_name": Var( + name="sim_name", + _type=str, + default="sim", + description="Name of the simulation.", + init_param=True, + init_assign=False, + init_super=True, + ), + "version": Var( + name="version", + _type=str, + default="mf6", + init_param=True, + init_assign=False, + init_super=True, + ), + "exe_name": Var( + name="exe_name", + _type=Union[str, PathLike], + default="mf6", + init_param=True, + init_assign=False, + init_super=True, + ), + "sim_ws": Var( + name="sim_ws", + _type=Union[str, PathLike], + default=os.curdir, + init_param=True, + init_assign=False, + init_super=True, + ), + "verbosity_level": Var( + name="verbosity_level", + _type=int, + default=1, + init_param=True, + init_assign=False, + init_super=True, + ), + "write_headers": Var( + name="write_headers", + _type=bool, + default=True, + init_param=True, + init_assign=False, + init_super=True, + ), + "use_pandas": Var( + name="use_pandas", + _type=bool, + default=True, + init_param=True, + init_assign=False, + init_super=True, + ), + "lazy_io": Var( + name="lazy_io", + _type=bool, + default=False, + init_param=True, + init_assign=False, + init_super=True, + ), + **vars_, + } + + # add initializer method parameters + # for this particular context type + if name.base is MFSimulation: + vars_ = _add_sim_params(vars_) + elif name.base is MFModel: + vars_ = _add_mdl_vars(vars_) + elif name.base is MFPackage: + if name.l == "exg": + vars_ = _add_exg_vars(vars_) + else: + vars_ = _add_pkg_vars(vars_) + + return vars_ + + def _metadata() -> List[Metadata]: + """ + Get a list of the class' original definition attributes, + as a partial, internal reproduction of the DFN contents. + + Notes + ----- + Currently, generated classes have a `.dfn` property that + reproduces the corresponding DFN sans a few attributes. + This represents the DFN in raw form, before adapting to + Python, consolidating nested types, etc. + """ + + def _to_dfn_fmt(var: Var) -> List[str]: + exclude = ["longname", "description"] + return [ + " ".join([k, v]) for k, v in var.items() if k not in exclude + ] + + return [["header"] + [attr for attr in (dfn.metadata or list())]] + [ + _to_dfn_fmt(var) for var in dfn.values() + ] + + return Context( + name=name, + base=name.base, + parent=name.parent, + description=name.description, + variables=_variables(), + metadata=_metadata(), + ) + + +def make_contexts( + dfn: Dfn, + common: Optional[Dfn] = None, + subpkgs: Optional[Subpkgs] = None, +) -> Iterator[Context]: + for name in dfn.name.contexts: + yield make_context(name=name, dfn=dfn, common=common, subpkgs=subpkgs) + + +_TEMPLATE_ENV = Environment( + loader=PackageLoader("flopy", "mf6/utils/templates/") +) + + +def make_targets( + dfn: Dfn, + outdir: Path, + common: Optional[Dfn] = None, + subpkgs: Optional[Subpkgs] = None, + verbose: bool = False, +): + """ + Generate Python source file(s) from the given input definition. + + Notes + ----- + + Model definitions will produce two files / classes, one for the + model itself and one for its corresponding control file package. + + All other definitions currently produce a single file and class. + """ + + template = _TEMPLATE_ENV.get_template("context.py.jinja") + for context in make_contexts(dfn=dfn, common=common, subpkgs=subpkgs): + target = outdir / context.name.target + with open(target, "w") as f: + source = template.render(**context.render()) + f.write(source) + if verbose: + print(f"Wrote {target}") + + +def make_all(dfndir: Path, outdir: Path, verbose: bool = False): + """Generate Python source files from the DFN files in the given location.""" + + # find definition files + paths = [ + p for p in dfndir.glob("*.dfn") if p.stem not in ["common", "flopy"] + ] + + # try to load common variables + common_path = dfndir / "common.dfn" + if not common_path.is_file: + warn("No common input definition file...") + common = None + else: + with open(common_path) as f: + common = load_dfn(f) + + # load all definitions first before we generate targets, + # so we can identify subpackages and create references + # between package/subpackage contexts. + dfns = dict() + subpkgs = dict() + for p in paths: + name = DfnName(*p.stem.split("-")) + with open(p) as f: + dfn = load_dfn(f, name=name) + dfns[name] = dfn + subpkg = Subpkg.from_dfn(dfn) + if subpkg: + # key is the name of the file record + # that corresponds to the subpackage + subpkgs[subpkg.key] = subpkg + + # generate target files + for name, dfn in dfns.items(): + with open(p) as f: + make_targets( + dfn=dfn, + outdir=outdir, + subpkgs=subpkgs, + common=common, + verbose=verbose, + ) + + # format the generated files + run_cmd("ruff", "format", outdir, verbose=verbose) + run_cmd("ruff", "check", "--fix", outdir, verbose=True) + + +_MF6_PATH = Path(__file__).parent +_DFN_PATH = _MF6_PATH / "data" / "dfn" +_TGT_PATH = _MF6_PATH / "modflow" + + +if __name__ == "__main__": + make_all(_DFN_PATH, _TGT_PATH) diff --git a/flopy/mf6/generate_classes.py b/flopy/mf6/utils/generate_classes.py similarity index 100% rename from flopy/mf6/generate_classes.py rename to flopy/mf6/utils/generate_classes.py diff --git a/flopy/mf6/utils/templates/attrs.jinja b/flopy/mf6/utils/templates/attrs.jinja new file mode 100644 index 0000000000..12f4b23131 --- /dev/null +++ b/flopy/mf6/utils/templates/attrs.jinja @@ -0,0 +1,15 @@ + {% for name, var in variables.items() if var.class_attr %} + {%- if var.kind == "list" or var.kind == "record" %} + {{ var.name }} = ListTemplateGenerator(("{{ component }}6", "{{ subcomponent }}", "{{ var.block }}", "{{ var.name }}")) + {%- elif var.kind == "array" %} + {{ var.name }} = ArrayTemplateGenerator(("{{ component }}6", "{{ subcomponent }}", "{{ var.block }}", "{{ var.name }}")) + {%- endif -%} + {%- endfor %} + {%- if base == "MFModel" %} + model_type = "{{ name.title }}" + {%- elif base == "MFPackage" %} + package_abbr = "{{ name.l }}{{ name.r }}" + _package_type = "{{ name.r }}" + dfn_file_name = "{{ name.l }}-{{ name.r }}.dfn" + dfn = {{ metadata|pprint|indent(10) }} + {% endif -%} \ No newline at end of file diff --git a/flopy/mf6/templates/context.jinja b/flopy/mf6/utils/templates/context.py.jinja similarity index 73% rename from flopy/mf6/templates/context.jinja rename to flopy/mf6/utils/templates/context.py.jinja index 53de8e174e..ced71a8990 100644 --- a/flopy/mf6/templates/context.jinja +++ b/flopy/mf6/utils/templates/context.py.jinja @@ -1,8 +1,8 @@ # autogenerated file, do not modify -from os import PathLike +from os import PathLike, curdir import typing import numpy as np -from typing import Any, Optional, Tuple, List, Dict, Union, Literal +from typing import Any, Optional, Tuple, List, Dict, Union, Literal, Iterable from numpy.typing import NDArray from flopy.mf6.data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenerator @@ -11,9 +11,11 @@ from flopy.mf6.mfmodel import MFModel from flopy.mf6 import MFSimulation, MFSimulationBase -class Modflow{{ class.title() }}({{ base }}): - """{% include "docstring.jinja" %}""" +class Modflow{{ name.title.title() }}({{ base }}): + {% include "docstring.jinja" %} {% include "attrs.jinja" %} {% include "init.jinja" %} + + {% include "load.jinja" %} \ No newline at end of file diff --git a/flopy/mf6/utils/templates/docstring.jinja b/flopy/mf6/utils/templates/docstring.jinja new file mode 100644 index 0000000000..488b567d45 --- /dev/null +++ b/flopy/mf6/utils/templates/docstring.jinja @@ -0,0 +1,12 @@ +""" + {{ description }} + + Parameters + ---------- + {% include "docstring_params.jinja" %} + + Methods + ------- + {% include "docstring_methods.jinja" %} + """ + diff --git a/flopy/mf6/utils/templates/docstring_methods.jinja b/flopy/mf6/utils/templates/docstring_methods.jinja new file mode 100644 index 0000000000..a050bf3576 --- /dev/null +++ b/flopy/mf6/utils/templates/docstring_methods.jinja @@ -0,0 +1,13 @@ +{% if base == "MFSimulation" %} + load : (sim_name : str, version : string, + exe_name : str or PathLike, sim_ws : str or PathLike, strict : bool, + verbosity_level : int, load_only : list, verify_data : bool, + write_headers : bool, lazy_io : bool, use_pandas : bool, + ) : MFSimulation + a class method that loads a simulation from files +{% elif base == "MFModel" %} + load : (simulation : MFSimulationData, model_name : string, + namfile : string, version : string, exe_name : string, + model_ws : string, strict : boolean) : MFSimulation + a class method that loads a model from files +{% endif %} \ No newline at end of file diff --git a/flopy/mf6/utils/templates/docstring_params.jinja b/flopy/mf6/utils/templates/docstring_params.jinja new file mode 100644 index 0000000000..2c4ba50f70 --- /dev/null +++ b/flopy/mf6/utils/templates/docstring_params.jinja @@ -0,0 +1,9 @@ +{% for var in variables.values() recursive %} + {% if loop.depth > 1 %}* {% endif %}{{ var.name }}{% if var._type is defined %} : {{ var._type }}{% endif %} +{%- if var.description is defined and not var.is_choice %} +{{ var.description|wordwrap|indent(4 + (loop.depth * 4), first=True) }} +{%- endif %} +{%- if var.children is defined -%} +{{ loop(var.children.values())|indent(4) }} +{%- endif %} +{% endfor %} \ No newline at end of file diff --git a/flopy/mf6/utils/templates/init.jinja b/flopy/mf6/utils/templates/init.jinja new file mode 100644 index 0000000000..352b0b3e24 --- /dev/null +++ b/flopy/mf6/utils/templates/init.jinja @@ -0,0 +1,72 @@ +def __init__( + self, + {%- for name, var in variables.items() if var.init_param %} + {%- if var.default is defined %} + {{ name }}{% if var._type is defined%}: {{ var._type }}{% endif %} = {{ var.default }}, + {%- else -%} + {{ name }}{% if var._type is defined%}: {{ var._type }}{% endif %}, + {% endif -%} + {%- endfor %} + **kwargs, + ): +{% if base == "MFSimulation" %} + super().__init__( + {%- for name, var in variables.items() if var.init_super %} + {{ name }}={{ name }}, + {%- endfor %} + **kwargs + ) + {%- for name, var in variables.items() if var.block == "options" %} + self.name_file.{{ name }}.set_data({{ name }}) + self.{{ name }} = self.name_file.{{ name }}, + {%- if var.subpkg is defined %} + self{{ var.subpkg.data_name }} = self._create_package( + "{{ var.subpkg.var_name }}", + self.{{ var.subpkg.data_name }} + ) + {% endif -%} + {% endfor -%} +{% elif base == "MFModel" %} + super().__init__( + model_type="{{ name.l }}6", + {%- for name, var in variables.items() if var.init_super %} + {{ name }}={{ name }}, + {%- endfor %} + **kwargs, + ) + {%- for name, var in variables.items() if var.block == "options" %} + self.name_file.{{ name }}.set_data({{ name }}) + self.{{ name }} = self.name_file.{{ name }} + {%- endfor %} +{% elif base == "MFPackage" %} + super().__init__( + {% if name.l == "exg" or name.l == "sln" -%} + simulation, + {%- endif %} + package_type="{{ name.r }}", + {%- for name, var in variables.items() if var.init_super %} + {{ name }}={{ name }}, + {%- endfor %} + **kwargs + ) + {% if name.l == "exg" -%} + simulation.register_exchange_file(self) + {%- endif %} + {% for name, var in variables.items() -%} + {%- if var.init_assign -%} + self.{{ name }} = {{ name }} + {% endif -%} + {%- if var.subpkg is defined -%} + self._{{ name }} = self.build_mfdata("{{ name }}", {{ name }}) + self._{{ var.subpkg.abbr }}_package = self.build_child_package( + "{{ var.subpkg.abbr }}", + {{ var.subpkg.val }}, + "{{ var.subpkg.param }}", + self.{{ var.subpkg.key }} + ) + {% elif var.init_build -%} + self.{{ name }} = self.build_mfdata("{{ name }}", {{ name }}) + {% endif -%} + {%- endfor -%} + self._init_complete = True +{% endif %} \ No newline at end of file diff --git a/flopy/mf6/utils/templates/load.jinja b/flopy/mf6/utils/templates/load.jinja new file mode 100644 index 0000000000..da8e59c58d --- /dev/null +++ b/flopy/mf6/utils/templates/load.jinja @@ -0,0 +1,58 @@ +{% if base == "MFSimulation" %} + @classmethod + def load( + cls, + sim_name="modflowsim", + version="mf6", + exe_name: Union[str, PathLike] = "mf6", + sim_ws: Union[str, PathLike] = curdir, + strict=True, + verbosity_level=1, + load_only=None, + verify_data=False, + write_headers=True, + lazy_io=False, + use_pandas=True, + ): + return MFSimulationBase.load( + cls, + sim_name, + version, + exe_name, + sim_ws, + strict, + verbosity_level, + load_only, + verify_data, + write_headers, + lazy_io, + use_pandas, + ) +{% elif base == "MFModel" %} + @classmethod + def load( + cls, + simulation, + structure, + modelname="NewModel", + model_nam_file="modflowtest.nam", + version="mf6", + exe_name="mf6", + strict=True, + model_rel_path=curdir, + load_only=None, + ): + return MFModel.load_base( + cls, + simulation, + structure, + modelname, + model_nam_file, + "{{ name.title }}6", + version, + exe_name, + strict, + model_rel_path, + load_only, + ) +{% endif %} \ No newline at end of file From 467859763719b4c0248cebfd934f295543dba52f Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 1 Oct 2024 16:24:55 -0400 Subject: [PATCH 13/46] cleanup --- .docs/code.rst | 4 ++-- .docs/md/generate_classes.md | 12 ++++++------ .github/workflows/commit.yml | 2 +- .github/workflows/examples.yml | 2 +- autotest/test_generate_classes.py | 2 +- docs/make_release.md | 2 +- docs/mf6_dev_guide.md | 2 +- flopy/mf6/utils/generate_classes.py | 2 +- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.docs/code.rst b/.docs/code.rst index f2d632154b..3bfeb040a6 100644 --- a/.docs/code.rst +++ b/.docs/code.rst @@ -231,8 +231,8 @@ Contents: .. toctree:: :maxdepth: 4 - ./source/flopy.mf6.createpackages.rst - ./source/flopy.mf6.generate_classes.rst + ./source/flopy.mf6.utils.createpackages.rst + ./source/flopy.mf6.utils.generate_classes.rst Previous Versions of MODFLOW diff --git a/.docs/md/generate_classes.md b/.docs/md/generate_classes.md index c480b9b62d..05c2300d40 100644 --- a/.docs/md/generate_classes.md +++ b/.docs/md/generate_classes.md @@ -13,7 +13,7 @@ MODFLOW 6 input continues to evolve as new models, packages, and options are dev The FloPy classes for MODFLOW 6 are largely generated by a utility which converts DFN files in a modflow6 repository on GitHub or on the local machine into Python source files in your local FloPy install. For instance (output much abbreviated): ```bash -$ python -m flopy.mf6.generate_classes +$ python -m flopy.mf6.utils.generate_classes @@ -55,7 +55,7 @@ Similar functionality is available within Python, e.g.: The `generate_classes()` function has several optional parameters. ```bash -$ python -m flopy.mf6.generate_classes -h +$ python -m flopy.mf6.utils.generate_classes -h usage: generate_classes.py [-h] [--owner OWNER] [--repo REPO] [--ref REF] [--dfnpath DFNPATH] [--no-backup] @@ -79,19 +79,19 @@ options: For example, use the develop branch instead: ```bash -$ python -m flopy.mf6.generate_classes --ref develop +$ python -m flopy.mf6.utils.generate_classes --ref develop ``` use a fork of modflow6: ```bash -$ python -m flopy.mf6.generate_classes --owner your-username --ref your-branch +$ python -m flopy.mf6.utils.generate_classes --owner your-username --ref your-branch ``` maybe your fork has a different name: ```bash -$ python -m flopy.mf6.generate_classes --owner your-username --repo your-modflow6 --ref your-branch +$ python -m flopy.mf6.utils.generate_classes --owner your-username --repo your-modflow6 --ref your-branch ``` local copy of the repo: ```bash -$ python -m flopy.mf6.generate_classes --dfnpath ../your/dfn/path +$ python -m flopy.mf6.utils.generate_classes --dfnpath ../your/dfn/path ``` Branch names, commit hashes, or tags may be provided to `ref`. diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml index e6c12ea268..07e7b73bcc 100644 --- a/.github/workflows/commit.yml +++ b/.github/workflows/commit.yml @@ -175,7 +175,7 @@ jobs: subset: triangle - name: Update package classes - run: python -m flopy.mf6.generate_classes --ref develop --no-backup + run: python -m flopy.mf6.utils.generate_classes --ref develop --no-backup - name: Run tests working-directory: autotest diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index fdf12b12f4..590a1578f3 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -69,7 +69,7 @@ jobs: subset: triangle - name: Update FloPy packages - run: python -m flopy.mf6.generate_classes --ref develop --no-backup + run: python -m flopy.mf6.utils.generate_classes --ref develop --no-backup - name: Run example tests working-directory: autotest diff --git a/autotest/test_generate_classes.py b/autotest/test_generate_classes.py index 44cac88ca1..db812aa40b 100644 --- a/autotest/test_generate_classes.py +++ b/autotest/test_generate_classes.py @@ -122,7 +122,7 @@ def test_generate_classes_from_github_refs( out, err, ret = run_cmd( str(python), "-m", - "flopy.mf6.generate_classes", + "flopy.mf6.utils.generate_classes", "--owner", owner, "--repo", diff --git a/docs/make_release.md b/docs/make_release.md index 568ebe763c..d65c7619f5 100644 --- a/docs/make_release.md +++ b/docs/make_release.md @@ -96,7 +96,7 @@ As described above, making a release manually involves the following steps: - Run `python scripts/update_version.py -v ` to update the version number stored in `version.txt` and `flopy/version.py`. For an approved release use the `--approve` flag. -- Update MODFLOW 6 dfn files in the repository and MODFLOW 6 package classes by running `python -m flopy.mf6.generate_classes --ref master --no-backup` +- Update MODFLOW 6 dfn files in the repository and MODFLOW 6 package classes by running `python -m flopy.mf6.utils.generate_classes --ref master --no-backup` - Run `ruff check .` and `ruff format .` from the project root. diff --git a/docs/mf6_dev_guide.md b/docs/mf6_dev_guide.md index 60aff50bc4..8f95ee9bbb 100644 --- a/docs/mf6_dev_guide.md +++ b/docs/mf6_dev_guide.md @@ -6,7 +6,7 @@ This file provides an overview of how FloPy for MODFLOW 6 (FPMF6) works under th Package Meta-Data and Package Files ----------------------------------------------- -FPMF6 uses meta-data files located in flopy/mf6/data/dfn to define the model and package types supported by MODFLOW 6. When additional model and package types are added to MODFLOW 6, additional meta-data files can be added to this folder and flopy/mf6/createpackages.py can be run to add new packages to the FloPy library. createpackages.py uses flopy/mf6/data/mfstructure.py to read meta-data files (*.dfn) and use that meta-data to create the package files found in flopy/mf6/modflow (do not directly modify any of the files in this folder, they are all automatically generated). The automatically generated package files contain an interface for accessing package data and data documentation generated from the meta-data files. Additionally, meta-data describing package data types and shapes is stored in the dfn attribute. flopy/mf6/data/mfstructure.py can load structure information using the dfn attribute (instead of loading it from the meta-data files). This allows for flopy to be installed without the dfn files. +FPMF6 uses meta-data files located in flopy/mf6/data/dfn to define the model and package types supported by MODFLOW 6. When additional model and package types are added to MODFLOW 6, additional meta-data files can be added to this folder and flopy/mf6/utils/createpackages.py can be run to add new packages to the FloPy library. createpackages.py uses flopy/mf6/data/mfstructure.py to read meta-data files (*.dfn) and use that meta-data to create the package files found in flopy/mf6/modflow (do not directly modify any of the files in this folder, they are all automatically generated). The automatically generated package files contain an interface for accessing package data and data documentation generated from the meta-data files. Additionally, meta-data describing package data types and shapes is stored in the dfn attribute. flopy/mf6/data/mfstructure.py can load structure information using the dfn attribute (instead of loading it from the meta-data files). This allows for flopy to be installed without the dfn files. All meta-data can be accessed from the flopy.mf6.data.mfstructure.MFStructure class. This is a singleton class, meaning only one instance of this class can be created. The class contains a sim_struct attribute (which is a flopy.mf6.data.mfstructure.MFSimulationStructure object) which contains all of the meta-data for all package files. Meta-data is stored in a structured format. MFSimulationStructure contains MFModelStructure and MFInputFileStructure objects, which contain the meta-data for each model type and each "simulation-level" package (tdis, ims, ...). MFModelStructure contains model specific meta-data and a MFInputFileStructure object for each package in that model. MFInputFileStructure contains package specific meta-data and a MFBlockStructure object for each block contained in the package file. MFBlockStructure contains block specific meta-data and a MFDataStructure object for each data structure defined in the block, and MFDataStructure contains data structure specific meta-data and a MFDataItemStructure object for each data item contained in the data structure. Data structures define the structure of data that is naturally grouped together, for example, the data in a numpy recarray. Data item structures define the structure of specific pieces of data, for example, a single column of a numpy recarray. The meta-data defined in these classes provides all the information FloPy needs to read and write MODFLOW 6 package and name files, create the Flopy interface, and check the data for various constraints. diff --git a/flopy/mf6/utils/generate_classes.py b/flopy/mf6/utils/generate_classes.py index 2763503d32..32c1d6978c 100644 --- a/flopy/mf6/utils/generate_classes.py +++ b/flopy/mf6/utils/generate_classes.py @@ -243,5 +243,5 @@ def cli_main(): if __name__ == "__main__": - """Run command-line with: python -m flopy.mf6.generate_classes""" + """Run command-line with: python -m flopy.mf6.utils.generate_classes""" cli_main() From a7fb655f4369e3f0664037c1493efbeb2f827f74 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 1 Oct 2024 17:44:59 -0400 Subject: [PATCH 14/46] fix subpkg support, more cleanup --- flopy/mf6/utils/createpackages.py | 98 +++++++++++++++++-- flopy/mf6/utils/templates/attrs.jinja | 10 +- .../utils/templates/docstring_methods.jinja | 2 +- .../utils/templates/docstring_params.jinja | 2 +- flopy/mf6/utils/templates/init.jinja | 28 +++--- flopy/mf6/utils/templates/load.jinja | 2 +- 6 files changed, 117 insertions(+), 25 deletions(-) diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index a144f49977..a71b98a6b7 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -281,7 +281,7 @@ def base(self) -> Optional[type]: """A base class from which the input context should inherit.""" l, r = self if self == ("sim", "nam"): - return MFSimulation + return MFSimulationBase if r is None: return MFModel return MFPackage @@ -442,6 +442,7 @@ class Subpkg: abbr: str param: str parents: List[type] + description: Optional[str] @classmethod def from_dfn(cls, dfn: Dfn) -> Optional["Subpkg"]: @@ -460,11 +461,13 @@ def from_dfn(cls, dfn: Dfn) -> Optional["Subpkg"]: def _subpkg(): line = lines["subpkg"] _, key, abbr, param, val = line.split() + descr = dfn.get(val, dict()).get("description", None) return { "key": key, "val": val, "abbr": abbr, "param": param, + "description": descr, } def _parents(): @@ -506,6 +509,7 @@ class VarKind(Enum): @classmethod def from_type(cls, t: type) -> Optional["VarKind"]: origin = get_origin(t) + args = get_args(t) if t is np.ndarray or origin is NDArray or origin is ArrayLike: return VarKind.Array elif origin is collections.abc.Iterable or origin is list: @@ -513,6 +517,8 @@ def from_type(cls, t: type) -> Optional["VarKind"]: elif origin is tuple: return VarKind.Record elif origin is Union: + if len(args) == 2 and args[1] is type(None): + return cls.from_type(args[0]) return VarKind.Union try: if issubclass(t, (bool, int, float, str)): @@ -527,7 +533,7 @@ class Var: """A variable in a MODFLOW 6 input context.""" name: str - _type: Optional[type] + _type: type block: Optional[str] description: Optional[str] default: Optional[Any] @@ -540,6 +546,7 @@ class Var: init_assign: bool = False init_build: bool = False init_super: bool = False + class_attr: bool = False def __init__( self, @@ -558,9 +565,10 @@ def __init__( init_assign: bool = False, init_build: bool = False, init_super: bool = False, + class_attr: bool = False, ): self.name = name - self._type = _type + self._type = _type or Any self.block = block self.description = description self.default = default @@ -589,6 +597,9 @@ def __init__( self.init_build = init_build # whether to pass arg to super().__init__() self.init_super = init_super + # whether the variable has a corresponding + # class attribute + self.class_attr = True Vars = Dict[str, Var] @@ -927,6 +938,7 @@ def _is_implicit_record(): # check if the variable references a subpackage subpkg = subpkgs.get(_name, None) if subpkg: + var_.init_build = False var_.subpkg = subpkg return var_ @@ -1056,8 +1068,13 @@ def _add_exg_vars(_vars: Vars) -> Vars: key = vars_.get(k, None) if not key: continue + vars_[subpkg.key].init_param = False + vars_[subpkg.key].class_attr = True + vars_[subpkg.key].subpkg = None vars_[subpkg.val] = Var( name=subpkg.val, + description=subpkg.description, + subpkg=subpkg, init_param=True, init_assign=False, init_super=False, @@ -1146,8 +1163,13 @@ def _add_pkg_vars(_vars: Vars) -> Vars: key = vars_.get(k, None) if not key: continue + vars_[subpkg.key].init_param = False + vars_[subpkg.key].class_attr = True + vars_[subpkg.key].subpkg = None vars_[subpkg.val] = Var( name=subpkg.val, + description=subpkg.description, + subpkg=subpkg, init_param=True, init_assign=False, init_super=False, @@ -1163,7 +1185,8 @@ def _add_mdl_vars(_vars: Vars) -> Vars: if packages: packages.init_param = False vars_["packages"] = packages - return { + + vars_ = { "simulation": Var( name="simulation", _type=MFSimulation, @@ -1225,6 +1248,36 @@ def _add_mdl_vars(_vars: Vars) -> Vars: **vars_, } + # if a reference map is provided, + # find any variables referring to + # subpackages, and attach another + # "value" variable for them all.. + # allows passing data directly to + # `__init__` instead of a path to + # load the subpackage from. maybe + # impossible if the data variable + # doesn't appear in the reference + # definition, though. + if subpkgs: + for k, subpkg in subpkgs.items(): + key = vars_.get(k, None) + if not key: + continue + vars_[subpkg.key].init_param = False + vars_[subpkg.key].class_attr = True + vars_[subpkg.key].subpkg = None + vars_[subpkg.val] = Var( + name=subpkg.val, + description=subpkg.description, + subpkg=subpkg, + init_param=True, + init_assign=False, + init_super=False, + init_build=False, + ) + + return vars_ + def _add_sim_params(_vars: Vars) -> Vars: """Add variables for a simulation context.""" vars_ = _vars.copy() @@ -1239,8 +1292,8 @@ def _add_sim_params(_vars: Vars) -> Vars: var = vars_.get(k, None) if var: var.init_param = False - vars_[k] = var - return { + vars_[k] = var + vars_ = { "sim_name": Var( name="sim_name", _type=str, @@ -1309,9 +1362,40 @@ def _add_sim_params(_vars: Vars) -> Vars: **vars_, } + # if a reference map is provided, + # find any variables referring to + # subpackages, and attach another + # "value" variable for them all.. + # allows passing data directly to + # `__init__` instead of a path to + # load the subpackage from. maybe + # impossible if the data variable + # doesn't appear in the reference + # definition, though. + if subpkgs: + for k, subpkg in subpkgs.items(): + key = vars_.get(k, None) + if not key: + continue + vars_[subpkg.key].init_param = False + vars_[subpkg.key].init_build = False + vars_[subpkg.key].class_attr = True + vars_[subpkg.key].subpkg = None + vars_[subpkg.param] = Var( + name=subpkg.param, + description=subpkg.description, + subpkg=subpkg, + init_param=True, + init_assign=False, + init_super=False, + init_build=False, + ) + + return vars_ + # add initializer method parameters # for this particular context type - if name.base is MFSimulation: + if name.base is MFSimulationBase: vars_ = _add_sim_params(vars_) elif name.base is MFModel: vars_ = _add_mdl_vars(vars_) diff --git a/flopy/mf6/utils/templates/attrs.jinja b/flopy/mf6/utils/templates/attrs.jinja index 12f4b23131..30c774905b 100644 --- a/flopy/mf6/utils/templates/attrs.jinja +++ b/flopy/mf6/utils/templates/attrs.jinja @@ -1,10 +1,12 @@ - {% for name, var in variables.items() if var.class_attr %} - {%- if var.kind == "list" or var.kind == "record" %} - {{ var.name }} = ListTemplateGenerator(("{{ component }}6", "{{ subcomponent }}", "{{ var.block }}", "{{ var.name }}")) + {%- if base != "MFSimulationBase" %} + {% for var in variables.values() if var.class_attr %} + {%- if var.kind == "list" or var.kind == "record" or var.kind == "union" %} + {{ var.name }} = ListTemplateGenerator(("{{ name.l }}6", "{{ name.r }}", "{{ var.block }}", "{{ var.name }}")) {%- elif var.kind == "array" %} - {{ var.name }} = ArrayTemplateGenerator(("{{ component }}6", "{{ subcomponent }}", "{{ var.block }}", "{{ var.name }}")) + {{ var.name }} = ArrayTemplateGenerator(("{{ name.l }}6", "{{ name.r }}", "{{ var.block }}", "{{ var.name }}")) {%- endif -%} {%- endfor %} + {% endif -%} {%- if base == "MFModel" %} model_type = "{{ name.title }}" {%- elif base == "MFPackage" %} diff --git a/flopy/mf6/utils/templates/docstring_methods.jinja b/flopy/mf6/utils/templates/docstring_methods.jinja index a050bf3576..41daf5715d 100644 --- a/flopy/mf6/utils/templates/docstring_methods.jinja +++ b/flopy/mf6/utils/templates/docstring_methods.jinja @@ -1,4 +1,4 @@ -{% if base == "MFSimulation" %} +{% if base == "MFSimulationBase" %} load : (sim_name : str, version : string, exe_name : str or PathLike, sim_ws : str or PathLike, strict : bool, verbosity_level : int, load_only : list, verify_data : bool, diff --git a/flopy/mf6/utils/templates/docstring_params.jinja b/flopy/mf6/utils/templates/docstring_params.jinja index 2c4ba50f70..56bb8750bf 100644 --- a/flopy/mf6/utils/templates/docstring_params.jinja +++ b/flopy/mf6/utils/templates/docstring_params.jinja @@ -1,5 +1,5 @@ {% for var in variables.values() recursive %} - {% if loop.depth > 1 %}* {% endif %}{{ var.name }}{% if var._type is defined %} : {{ var._type }}{% endif %} + {% if loop.depth > 1 %}* {% endif %}{{ var.name }} : {{ var._type }} {%- if var.description is defined and not var.is_choice %} {{ var.description|wordwrap|indent(4 + (loop.depth * 4), first=True) }} {%- endif %} diff --git a/flopy/mf6/utils/templates/init.jinja b/flopy/mf6/utils/templates/init.jinja index 352b0b3e24..9681011513 100644 --- a/flopy/mf6/utils/templates/init.jinja +++ b/flopy/mf6/utils/templates/init.jinja @@ -2,27 +2,29 @@ def __init__( self, {%- for name, var in variables.items() if var.init_param %} {%- if var.default is defined %} - {{ name }}{% if var._type is defined%}: {{ var._type }}{% endif %} = {{ var.default }}, + {{ name }}: {{ var._type }} = {{ var.default }}, {%- else -%} - {{ name }}{% if var._type is defined%}: {{ var._type }}{% endif %}, + {{ name }}: {{ var._type }}, {% endif -%} {%- endfor %} **kwargs, ): -{% if base == "MFSimulation" %} +{% if base == "MFSimulationBase" %} super().__init__( {%- for name, var in variables.items() if var.init_super %} {{ name }}={{ name }}, {%- endfor %} **kwargs ) - {%- for name, var in variables.items() if var.block == "options" %} + {%- for name, var in variables.items() %} + {%- if var.block == "options" and var.init_build %} self.name_file.{{ name }}.set_data({{ name }}) - self.{{ name }} = self.name_file.{{ name }}, + self.{{ name }} = self.name_file.{{ name }} + {% endif -%} {%- if var.subpkg is defined %} - self{{ var.subpkg.data_name }} = self._create_package( - "{{ var.subpkg.var_name }}", - self.{{ var.subpkg.data_name }} + self.{{ var.subpkg.param }} = self._create_package( + "{{ var.subpkg.abbr }}", + {{ var.subpkg.param }} ) {% endif -%} {% endfor -%} @@ -34,14 +36,18 @@ def __init__( {%- endfor %} **kwargs, ) - {%- for name, var in variables.items() if var.block == "options" %} + {%- for name, var in variables.items() %} + {%- if var.block == "options" and var.init_build %} self.name_file.{{ name }}.set_data({{ name }}) self.{{ name }} = self.name_file.{{ name }} + {% endif -%} {%- endfor %} {% elif base == "MFPackage" %} super().__init__( - {% if name.l == "exg" or name.l == "sln" -%} + {% if parent == "MFSimulation" -%} simulation, + {% elif parent == "MFModel" -%} + model, {%- endif %} package_type="{{ name.r }}", {%- for name, var in variables.items() if var.init_super %} @@ -57,7 +63,7 @@ def __init__( self.{{ name }} = {{ name }} {% endif -%} {%- if var.subpkg is defined -%} - self._{{ name }} = self.build_mfdata("{{ name }}", {{ name }}) + self._{{ name }} = self.build_mfdata("{{ name }}", None) self._{{ var.subpkg.abbr }}_package = self.build_child_package( "{{ var.subpkg.abbr }}", {{ var.subpkg.val }}, diff --git a/flopy/mf6/utils/templates/load.jinja b/flopy/mf6/utils/templates/load.jinja index da8e59c58d..5ba8cc9f89 100644 --- a/flopy/mf6/utils/templates/load.jinja +++ b/flopy/mf6/utils/templates/load.jinja @@ -1,4 +1,4 @@ -{% if base == "MFSimulation" %} +{% if base == "MFSimulationBase" %} @classmethod def load( cls, From c056dab4edb8f9222f24c743f230f21006569b13 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 1 Oct 2024 19:22:11 -0400 Subject: [PATCH 15/46] named tuples for records --- flopy/mf6/utils/createpackages.py | 65 +++++++++++++++++----- flopy/mf6/utils/templates/context.py.jinja | 3 + flopy/mf6/utils/templates/records.jinja | 3 + 3 files changed, 56 insertions(+), 15 deletions(-) create mode 100644 flopy/mf6/utils/templates/records.jinja diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index a71b98a6b7..8b308bef29 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -83,7 +83,7 @@ import collections import os -from collections import UserDict +from collections import UserDict, namedtuple from dataclasses import asdict, dataclass, replace from enum import Enum from keyword import kwlist @@ -627,8 +627,9 @@ class Context: base: Optional[type] parent: Optional[type] description: Optional[str] - variables: Vars metadata: Metadata + variables: Vars + records: Vars _SCALAR_TYPES = { @@ -667,8 +668,12 @@ def make_context( is related to. """ - subpkgs = subpkgs or dict() common = common or dict() + subpkgs = subpkgs or dict() + records = dict() + + def _nt_name(s): + return s.title().replace("record", "").replace("-", "_") def _convert( var: Dict[str, str], @@ -708,6 +713,7 @@ def _convert( tagged = var.get("tagged, False") description = var.get("description", "") children = None + is_record = False def _description(descr: str) -> str: """ @@ -823,15 +829,27 @@ def _is_implicit_record(): elif _is_implicit_record(): record_name = _name record_fields = _fields(record_name) - record_type = Tuple[ - tuple([f._type for f in record_fields.values()]) - ] + field_types = [f._type for f in record_fields.values()] + record_type = Tuple[tuple(field_types)] record = Var( name=record_name, _type=record_type, block=block, children=record_fields, ) + records[_nt_name(record_name)] = replace( + record, name=_nt_name(record_name) + ) + # TODO: do we want to use named tuples here? + # it's a bit less explicit and requires looking + # back and forth between the tuple definition + # and the class docstring... but nice to have + # a concise definition of each record type... + record_type = namedtuple( + _nt_name(record_name), + [_nt_name(k) for k in record_fields.keys()], + ) + record = replace(record, _type=record_type) children = {record_name: record} type_ = Iterable[record_type] else: @@ -861,6 +879,7 @@ def _is_implicit_record(): record_type = t if get_origin(t) is tuple else Tuple[(t,)] # TODO: if record has 1 field, accept value directly? type_ = record_type + is_record = True # are we wrapping a record which is a # choice within a union? if so, use a @@ -881,6 +900,7 @@ def _is_implicit_record(): field_name: replace(field, _type=field_type, is_choice=True) } type_ = record_type + is_record = True # at this point, if it has a shape, it's an array.. # but if it's in a record make it a variadic tuple, @@ -906,14 +926,6 @@ def _is_implicit_record(): tag = _type == "keyword" and (tagged or wrap) type_ = Literal[_name] if tag else _SCALAR_TYPES.get(_type, _type) - # make optional if needed - if optional: - type_ = ( - Optional[type_] - if (type_ is not bool and not in_record and not wrap) - else type_ - ) - # format the variable description description = _description(description) @@ -941,6 +953,27 @@ def _is_implicit_record(): var_.init_build = False var_.subpkg = subpkg + # make named tuples for record types + if is_record: + records[_nt_name(name_)] = replace(var_, name=_nt_name(name_)) + # TODO: do we want to use named tuples here? + # it's a bit less explicit and requires looking + # back and forth between the tuple definition + # and the class docstring... but nice to have + # a concise definition of each record type... + if children: + type_ = namedtuple( + _nt_name(name_), [_nt_name(k) for k in children.keys()] + ) + + # make optional if needed + if optional: + var_._type = ( + Optional[type_] + if (type_ is not bool and not in_record and not wrap) + else type_ + ) + return var_ def _variables() -> Vars: @@ -965,6 +998,7 @@ def _variables() -> Vars: # their parents in the hierarchy if not var.get("in_record", False) } + # set the name since we may have altered # it when creating the variable (e.g. to # avoid name/reserved keyword collisions. @@ -1435,8 +1469,9 @@ def _to_dfn_fmt(var: Var) -> List[str]: base=name.base, parent=name.parent, description=name.description, - variables=_variables(), metadata=_metadata(), + variables=_variables(), + records=records, ) diff --git a/flopy/mf6/utils/templates/context.py.jinja b/flopy/mf6/utils/templates/context.py.jinja index ced71a8990..baeb24f2b3 100644 --- a/flopy/mf6/utils/templates/context.py.jinja +++ b/flopy/mf6/utils/templates/context.py.jinja @@ -11,6 +11,9 @@ from flopy.mf6.mfmodel import MFModel from flopy.mf6 import MFSimulation, MFSimulationBase +{% include "records.jinja" %} + + class Modflow{{ name.title.title() }}({{ base }}): {% include "docstring.jinja" %} diff --git a/flopy/mf6/utils/templates/records.jinja b/flopy/mf6/utils/templates/records.jinja new file mode 100644 index 0000000000..f2ebfbcc2d --- /dev/null +++ b/flopy/mf6/utils/templates/records.jinja @@ -0,0 +1,3 @@ +{% for name, var in records.items() -%} +{{ name }} = {{ var._type }} +{% endfor -%} \ No newline at end of file From 09d3dca7eecc3d13f1976ab85862bb579be21938 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 1 Oct 2024 19:23:40 -0400 Subject: [PATCH 16/46] py39 back-compat... --- flopy/mf6/utils/createpackages.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index 8b308bef29..8f92cfa6f5 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -125,17 +125,17 @@ def _try_get_type_name(t) -> str: args = get_args(t) if origin is Literal: args = ['"' + a + '"' for a in args] - return f"{Literal.__name__}[{', '.join(args)}]" + return f"Literal[{', '.join(args)}]" elif origin is Union: if len(args) == 2 and args[1] is type(None): return f"{Optional.__name__}[{_try_get_type_name(args[0])}]" - return f"{Union.__name__}[{', '.join([_try_get_type_name(a) for a in args])}]" + return f"Union[{', '.join([_try_get_type_name(a) for a in args])}]" elif origin is tuple: - return f"{Tuple.__name__}[{', '.join([_try_get_type_name(a) for a in args])}]" + return f"Tuple[{', '.join([_try_get_type_name(a) for a in args])}]" elif origin is collections.abc.Iterable: - return f"{Iterable.__name__}[{', '.join([_try_get_type_name(a) for a in args])}]" + return f"Iterable[{', '.join([_try_get_type_name(a) for a in args])}]" elif origin is list: - return f"{List.__name__}[{', '.join([_try_get_type_name(a) for a in args])}]" + return f"List[{', '.join([_try_get_type_name(a) for a in args])}]" elif origin is np.ndarray: return f"NDArray[np.{_try_get_type_name(args[1].__args__[0])}]" elif origin is np.dtype: From 76ef6530c03da54cb4aeab292507a0b226832ece Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 1 Oct 2024 19:31:14 -0400 Subject: [PATCH 17/46] py39 --- flopy/mf6/utils/createpackages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index 8f92cfa6f5..03c5b4551b 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -128,7 +128,7 @@ def _try_get_type_name(t) -> str: return f"Literal[{', '.join(args)}]" elif origin is Union: if len(args) == 2 and args[1] is type(None): - return f"{Optional.__name__}[{_try_get_type_name(args[0])}]" + return f"Optional[{_try_get_type_name(args[0])}]" return f"Union[{', '.join([_try_get_type_name(a) for a in args])}]" elif origin is tuple: return f"Tuple[{', '.join([_try_get_type_name(a) for a in args])}]" From f09086b88aadd62e3eca3110752c17d844be3ab4 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 1 Oct 2024 20:15:31 -0400 Subject: [PATCH 18/46] generate_classes usage --- flopy/mf6/utils/createpackages.py | 3 +-- flopy/mf6/utils/generate_classes.py | 9 +++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index 03c5b4551b..d5711ea201 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -112,8 +112,6 @@ from numpy.typing import ArrayLike, NDArray from flopy.mf6 import MFSimulation - -# keep below as absolute imports from flopy.mf6.mfmodel import MFModel from flopy.mf6.mfpackage import MFPackage from flopy.mf6.mfsimbase import MFSimulationBase @@ -673,6 +671,7 @@ def make_context( records = dict() def _nt_name(s): + """Trim name of a named tuple representing a record.""" return s.title().replace("record", "").replace("-", "_") def _convert( diff --git a/flopy/mf6/utils/generate_classes.py b/flopy/mf6/utils/generate_classes.py index 32c1d6978c..44e52031ca 100644 --- a/flopy/mf6/utils/generate_classes.py +++ b/flopy/mf6/utils/generate_classes.py @@ -2,9 +2,10 @@ import shutil import tempfile import time +from pathlib import Path from warnings import warn -from .createpackages import create_packages +from .createpackages import make_all thisfilepath = os.path.dirname(os.path.abspath(__file__)) flopypth = os.path.join(thisfilepath, "..", "..") @@ -14,6 +15,10 @@ default_owner = "MODFLOW-USGS" default_repo = "modflow6" +_MF6_PATH = Path(__file__).parent +_DFN_PATH = _MF6_PATH / "data" / "dfn" +_TGT_PATH = _MF6_PATH / "modflow" + def delete_files(files, pth, allow_failure=False, exclude=None): if exclude is None: @@ -189,7 +194,7 @@ def generate_classes( delete_mf6_classes() print(" Create mf6 classes using the downloaded definition files.") - create_packages() + make_all(_DFN_PATH, _TGT_PATH) list_files(os.path.join(flopypth, "mf6", "modflow")) From 0e6aca80d24f72163de9385184798a8ae8c9e10d Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 1 Oct 2024 21:20:16 -0400 Subject: [PATCH 19/46] fixes --- flopy/mf6/utils/createpackages.py | 74 ++++++++++++---------- flopy/mf6/utils/generate_classes.py | 2 +- flopy/mf6/utils/templates/context.py.jinja | 7 +- 3 files changed, 45 insertions(+), 38 deletions(-) diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index d5711ea201..baaa8b332a 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -111,11 +111,6 @@ from modflow_devtools.misc import run_cmd from numpy.typing import ArrayLike, NDArray -from flopy.mf6 import MFSimulation -from flopy.mf6.mfmodel import MFModel -from flopy.mf6.mfpackage import MFPackage -from flopy.mf6.mfsimbase import MFSimulationBase - def _try_get_type_name(t) -> str: """Convert a type to a name suitable for templating.""" @@ -125,7 +120,9 @@ def _try_get_type_name(t) -> str: args = ['"' + a + '"' for a in args] return f"Literal[{', '.join(args)}]" elif origin is Union: - if len(args) == 2 and args[1] is type(None): + if len(args) >= 2 and args[-1] is type(None): + if len(args) > 2: + return f"Optional[Tuple[{', '.join([_try_get_type_name(a) for a in args[:-1]])}]]" return f"Optional[{_try_get_type_name(args[0])}]" return f"Union[{', '.join([_try_get_type_name(a) for a in args])}]" elif origin is tuple: @@ -193,12 +190,15 @@ def renderable( def __renderable(cls): def _render(d: dict) -> dict: + def _render_key(k): + return k + def _render_val(v): return _try_get_type_name(_try_get_enum_value(v)) # drop nones except for any explicitly kept _d = { - k: _render_val(v) + _render_key(k): _render_val(v) for k, v in d.items() if (k in keep_none or v is not None) } @@ -275,26 +275,26 @@ def title(self) -> str: return f"{l}{r}" @property - def base(self) -> Optional[type]: + def base(self) -> Optional[str]: """A base class from which the input context should inherit.""" l, r = self if self == ("sim", "nam"): - return MFSimulationBase + return "MFSimulationBase" if r is None: - return MFModel - return MFPackage + return "MFModel" + return "MFPackage" @property - def parent(self) -> Optional[type]: + def parent(self) -> Optional[str]: """A parent context which owns and manages the input context.""" l, r = self if (l, r) == ("sim", "nam"): return None if l in ["sim", "exg", "sln"]: - return MFSimulation + return "MFSimulation" if l == "utl": - return MFPackage - return MFModel + return "MFPackage" + return "MFModel" @property def target(self) -> str: @@ -306,14 +306,14 @@ def description(self) -> str: """A description of the input context.""" l, r = self title = self.title.title() - if self.base == MFPackage: + if self.base == "MFPackage": return ( f"Modflow{title} defines a {r} package within a {l} " - f"{self.base.__name__.lower().replace('mf', '')}." + f"{self.base.lower().replace('mf', '')}." ) - elif self.base == MFModel: + elif self.base == "MFModel": return f"Modflow{title} defines a {l.upper()} model." - elif self.base == MFSimulation: + elif self.base == "MFSimulation": return """ MFSimulation is used to load, build, and/or save a MODFLOW 6 simulation. A MFSimulation object must be created before creating any of the MODFLOW 6 @@ -475,10 +475,10 @@ def _parents(): def _parent(t): if "simulation" in t: - return MFSimulation + return "MFSimulation" elif "model" in t: - return MFModel - return MFPackage + return "MFModel" + return "MFPackage" return [_parent(pt) for pt in _type.split("/")] @@ -531,7 +531,7 @@ class Var: """A variable in a MODFLOW 6 input context.""" name: str - _type: type + _type: Union[type, str] block: Optional[str] description: Optional[str] default: Optional[Any] @@ -672,7 +672,10 @@ def make_context( def _nt_name(s): """Trim name of a named tuple representing a record.""" - return s.title().replace("record", "").replace("-", "_") + s = s.title().replace("record", "").replace("-", "_") + if s.endswith("s"): + return s[:-1] + return s def _convert( var: Dict[str, str], @@ -823,7 +826,7 @@ def _is_implicit_record(): record_name = names[0] record_spec = dfn[record_name] record_type = _convert(record_spec, wrap=False) - children = {record_name: record_type} + children = {_nt_name(record_name).lower(): record_type} type_ = Iterable[record_type._type] elif _is_implicit_record(): record_name = _name @@ -848,8 +851,8 @@ def _is_implicit_record(): _nt_name(record_name), [_nt_name(k) for k in record_fields.keys()], ) - record = replace(record, _type=record_type) - children = {record_name: record} + record = replace(record, _type=record_type, name=_nt_name(record_name).lower()) + children = {_nt_name(record_name): record} type_ = Iterable[record_type] else: # irregular recarray, rows can be any of several types @@ -933,6 +936,7 @@ def _is_implicit_record(): # if name is a reserved keyword, add a trailing underscore to it name_ = f"{_name}_" if _name in kwlist else _name + name_ = name_.replace("-", "_") # create var var_ = Var( @@ -1015,7 +1019,7 @@ def _add_exg_vars(_vars: Vars) -> Vars: vars_ = { "simulation": Var( name="simulation", - _type=MFSimulation, + _type="MFSimulation", description=( "Simulation that this package is a part of. " "Package is automatically added to simulation " @@ -1119,7 +1123,7 @@ def _add_exg_vars(_vars: Vars) -> Vars: def _add_pkg_vars(_vars: Vars) -> Vars: """Add variables for a package context.""" parent_type = name.parent - parent_name = parent_type.__name__.lower().replace("mf", "") + parent_name = parent_type.lower().replace("mf", "") vars_ = { parent_name: Var( name=parent_name, @@ -1222,7 +1226,7 @@ def _add_mdl_vars(_vars: Vars) -> Vars: vars_ = { "simulation": Var( name="simulation", - _type=MFSimulation, + _type="MFSimulation", description=( "Simulation that this model is part of. " "Model is automatically added to the simulation " @@ -1428,11 +1432,11 @@ def _add_sim_params(_vars: Vars) -> Vars: # add initializer method parameters # for this particular context type - if name.base is MFSimulationBase: + if name.base == "MFSimulationBase": vars_ = _add_sim_params(vars_) - elif name.base is MFModel: + elif name.base == "MFModel": vars_ = _add_mdl_vars(vars_) - elif name.base is MFPackage: + elif name.base == "MFPackage": if name.l == "exg": vars_ = _add_exg_vars(vars_) else: @@ -1531,7 +1535,7 @@ def make_all(dfndir: Path, outdir: Path, verbose: bool = False): warn("No common input definition file...") common = None else: - with open(common_path) as f: + with open(common_path, "r") as f: common = load_dfn(f) # load all definitions first before we generate targets, @@ -1566,7 +1570,7 @@ def make_all(dfndir: Path, outdir: Path, verbose: bool = False): run_cmd("ruff", "check", "--fix", outdir, verbose=True) -_MF6_PATH = Path(__file__).parent +_MF6_PATH = Path(__file__).parents[1] _DFN_PATH = _MF6_PATH / "data" / "dfn" _TGT_PATH = _MF6_PATH / "modflow" diff --git a/flopy/mf6/utils/generate_classes.py b/flopy/mf6/utils/generate_classes.py index 44e52031ca..5f59135171 100644 --- a/flopy/mf6/utils/generate_classes.py +++ b/flopy/mf6/utils/generate_classes.py @@ -15,7 +15,7 @@ default_owner = "MODFLOW-USGS" default_repo = "modflow6" -_MF6_PATH = Path(__file__).parent +_MF6_PATH = Path(__file__).parents[1] _DFN_PATH = _MF6_PATH / "data" / "dfn" _TGT_PATH = _MF6_PATH / "modflow" diff --git a/flopy/mf6/utils/templates/context.py.jinja b/flopy/mf6/utils/templates/context.py.jinja index baeb24f2b3..f9edd9cd76 100644 --- a/flopy/mf6/utils/templates/context.py.jinja +++ b/flopy/mf6/utils/templates/context.py.jinja @@ -8,13 +8,16 @@ from numpy.typing import NDArray from flopy.mf6.data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenerator from flopy.mf6.mfpackage import MFPackage from flopy.mf6.mfmodel import MFModel -from flopy.mf6 import MFSimulation, MFSimulationBase +from flopy.mf6.mfsimbase import MFSimulationBase +{% if base != "MFSimulationBase" %} +from flopy.mf6.modflow.mfsimulation import MFSimulation +{% endif %} {% include "records.jinja" %} -class Modflow{{ name.title.title() }}({{ base }}): +class {% if base == "MFSimulationBase" %}MF{% else %}Modflow{% endif %}{{ name.title.title() }}({{ base }}): {% include "docstring.jinja" %} {% include "attrs.jinja" %} From 38763160d54c460d12d13ca03625d8b10afb8a15 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 1 Oct 2024 21:20:25 -0400 Subject: [PATCH 20/46] ruff --- flopy/mf6/utils/createpackages.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index baaa8b332a..a7693fedc8 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -851,7 +851,11 @@ def _is_implicit_record(): _nt_name(record_name), [_nt_name(k) for k in record_fields.keys()], ) - record = replace(record, _type=record_type, name=_nt_name(record_name).lower()) + record = replace( + record, + _type=record_type, + name=_nt_name(record_name).lower(), + ) children = {_nt_name(record_name): record} type_ = Iterable[record_type] else: From 6893f64ecab7ec533f126fb68882cac63dc6ea89 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 2 Oct 2024 09:58:46 -0400 Subject: [PATCH 21/46] subpkg fixes --- flopy/mf6/data/dfn/utl-tas.dfn | 4 + flopy/mf6/utils/createpackages.py | 97 ++++++++++++---------- flopy/mf6/utils/templates/context.py.jinja | 2 - flopy/mf6/utils/templates/init.jinja | 8 +- flopy/mf6/utils/templates/records.jinja | 2 +- 5 files changed, 65 insertions(+), 48 deletions(-) diff --git a/flopy/mf6/data/dfn/utl-tas.dfn b/flopy/mf6/data/dfn/utl-tas.dfn index 6316beba5c..59d47f1422 100644 --- a/flopy/mf6/data/dfn/utl-tas.dfn +++ b/flopy/mf6/data/dfn/utl-tas.dfn @@ -48,6 +48,7 @@ type keyword shape reader urword optional false +in_record true longname description xxx @@ -59,6 +60,7 @@ shape tagged false reader urword optional false +in_record true longname description Interpolation method, which is either STEPWISE or LINEAR. @@ -78,6 +80,7 @@ type keyword shape reader urword optional false +in_record true longname description xxx @@ -88,6 +91,7 @@ shape time_series_name tagged false reader urword optional false +in_record true longname description Scale factor, which will multiply all array values in time series. SFAC is an optional attribute; if omitted, SFAC = 1.0. diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index a7693fedc8..bfa1468677 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -284,18 +284,6 @@ def base(self) -> Optional[str]: return "MFModel" return "MFPackage" - @property - def parent(self) -> Optional[str]: - """A parent context which owns and manages the input context.""" - l, r = self - if (l, r) == ("sim", "nam"): - return None - if l in ["sim", "exg", "sln"]: - return "MFSimulation" - if l == "utl": - return "MFPackage" - return "MFModel" - @property def target(self) -> str: """The source file name to generate.""" @@ -439,7 +427,7 @@ class Subpkg: val: str abbr: str param: str - parents: List[type] + parents: List[Union[type, str]] description: Optional[str] @classmethod @@ -471,16 +459,7 @@ def _subpkg(): def _parents(): line = lines["parent"] _, _, _type = line.split() - _type = _type.lower() - - def _parent(t): - if "simulation" in t: - return "MFSimulation" - elif "model" in t: - return "MFModel" - return "MFPackage" - - return [_parent(pt) for pt in _type.split("/")] + return _type.split("/") return ( cls(**_subpkg(), parents=_parents()) @@ -508,16 +487,16 @@ class VarKind(Enum): def from_type(cls, t: type) -> Optional["VarKind"]: origin = get_origin(t) args = get_args(t) + if origin is Union: + if len(args) >= 2 and args[-1] is type(None): + return cls.from_type(args[0]) + return VarKind.Union if t is np.ndarray or origin is NDArray or origin is ArrayLike: return VarKind.Array elif origin is collections.abc.Iterable or origin is list: return VarKind.List elif origin is tuple: return VarKind.Record - elif origin is Union: - if len(args) == 2 and args[1] is type(None): - return cls.from_type(args[0]) - return VarKind.Union try: if issubclass(t, (bool, int, float, str)): return VarKind.Scalar @@ -544,7 +523,7 @@ class Var: init_assign: bool = False init_build: bool = False init_super: bool = False - class_attr: bool = False + class_attr: bool = True def __init__( self, @@ -563,7 +542,7 @@ def __init__( init_assign: bool = False, init_build: bool = False, init_super: bool = False, - class_attr: bool = False, + class_attr: bool = True, ): self.name = name self._type = _type or Any @@ -597,7 +576,7 @@ def __init__( self.init_super = init_super # whether the variable has a corresponding # class attribute - self.class_attr = True + self.class_attr = class_attr Vars = Dict[str, Var] @@ -671,12 +650,39 @@ def make_context( records = dict() def _nt_name(s): - """Trim name of a named tuple representing a record.""" - s = s.title().replace("record", "").replace("-", "_") - if s.endswith("s"): - return s[:-1] + """Trim the name of a record for a corresponding named tuple.""" + s = s.title().replace("record", "").replace("-", "_").replace("_", "") + # if s.endswith("s"): + # return s[:-1] return s + def _parent() -> Optional[str]: + """ + Get the context's parent(s), i.e. context(s) which can + own an instance of this context. If this context is a + subpackage which can have multiple parent types, this + will be a Union of possible parent types, otherwise a + single parent type. + + We return a string directly instead of a type to avoid + the need to import `MFSimulation/MFModel/MFPackage`. + """ + l, r = dfn.name + if (l, r) == ("sim", "nam"): + return None + if l in ["sim", "exg", "sln"]: + return "MFSimulation" + if r == "nam": + return "MFModel" + subpkg = Subpkg.from_dfn(dfn) + if subpkg: + if len(subpkg.parents) > 1: + return f"Union[{', '.join([_try_get_type_name(t) for t in subpkg.parents])}]" + return subpkg.parents[0] + return "MFPackage" + + parent = _parent() + def _convert( var: Dict[str, str], wrap: bool = False, @@ -877,7 +883,7 @@ def _is_implicit_record(): children = _fields(_name) if len(children) > 1: record_type = Tuple[ - tuple([c._type for c in children.values()]) + tuple([f._type for f in children.values()]) ] elif len(children) == 1: t = list(children.values())[0]._type @@ -939,8 +945,7 @@ def _is_implicit_record(): default = var.get("default", False if type_ is bool else None) # if name is a reserved keyword, add a trailing underscore to it - name_ = f"{_name}_" if _name in kwlist else _name - name_ = name_.replace("-", "_") + name_ = (f"{_name}_" if _name in kwlist else _name).replace("-", "_") # create var var_ = Var( @@ -972,6 +977,7 @@ def _is_implicit_record(): type_ = namedtuple( _nt_name(name_), [_nt_name(k) for k in children.keys()] ) + var_._type = type_ # make optional if needed if optional: @@ -1126,12 +1132,17 @@ def _add_exg_vars(_vars: Vars) -> Vars: def _add_pkg_vars(_vars: Vars) -> Vars: """Add variables for a package context.""" - parent_type = name.parent - parent_name = parent_type.lower().replace("mf", "") + parent_name = "parent_" + ( + parent.lower() + .replace("mf", "") + .replace("union[", "") + .replace("]", "") + .replace(", ", "_or_") + ) vars_ = { parent_name: Var( name=parent_name, - _type=parent_type, + _type=parent, description=( f"{parent_name.title()} that this package is part of. " f"Package is automatically added to the {parent_name} " @@ -1139,7 +1150,7 @@ def _add_pkg_vars(_vars: Vars) -> Vars: ), init_param=True, init_assign=False, - init_super=False, + init_super=True, ), "loading_package": Var( name="loading_package", @@ -1176,7 +1187,7 @@ def _add_pkg_vars(_vars: Vars) -> Vars: # is the path to the subpackage's # parent context subpkg = Subpkg.from_dfn(dfn) - if subpkg: + if subpkg and dfn.name.l != "utl": vars_["parent_file"] = Var( name="parent_file", _type=Union[str, PathLike], @@ -1474,7 +1485,7 @@ def _to_dfn_fmt(var: Var) -> List[str]: return Context( name=name, base=name.base, - parent=name.parent, + parent=parent, description=name.description, metadata=_metadata(), variables=_variables(), diff --git a/flopy/mf6/utils/templates/context.py.jinja b/flopy/mf6/utils/templates/context.py.jinja index f9edd9cd76..f49e2270fd 100644 --- a/flopy/mf6/utils/templates/context.py.jinja +++ b/flopy/mf6/utils/templates/context.py.jinja @@ -13,10 +13,8 @@ from flopy.mf6.mfsimbase import MFSimulationBase from flopy.mf6.modflow.mfsimulation import MFSimulation {% endif %} - {% include "records.jinja" %} - class {% if base == "MFSimulationBase" %}MF{% else %}Modflow{% endif %}{{ name.title.title() }}({{ base }}): {% include "docstring.jinja" %} diff --git a/flopy/mf6/utils/templates/init.jinja b/flopy/mf6/utils/templates/init.jinja index 9681011513..90c166c07a 100644 --- a/flopy/mf6/utils/templates/init.jinja +++ b/flopy/mf6/utils/templates/init.jinja @@ -48,10 +48,14 @@ def __init__( simulation, {% elif parent == "MFModel" -%} model, - {%- endif %} + {% endif -%} package_type="{{ name.r }}", {%- for name, var in variables.items() if var.init_super %} - {{ name }}={{ name }}, + {% if "parent" in name -%} + parent={{ name }}, + {%- else -%} + {{ name|replace("_package", "") }}={{ name }}, + {%- endif -%} {%- endfor %} **kwargs ) diff --git a/flopy/mf6/utils/templates/records.jinja b/flopy/mf6/utils/templates/records.jinja index f2ebfbcc2d..a12f855cbe 100644 --- a/flopy/mf6/utils/templates/records.jinja +++ b/flopy/mf6/utils/templates/records.jinja @@ -1,3 +1,3 @@ -{% for name, var in records.items() -%} +{%- for name, var in records.items() -%} {{ name }} = {{ var._type }} {% endfor -%} \ No newline at end of file From bf3f2ee97d139a7c33dd1862e9addd1dfa2e4776 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 2 Oct 2024 10:21:55 -0400 Subject: [PATCH 22/46] MFChildPackages container for subpkgs --- flopy/mf6/data/dfn/utl-tas.dfn | 2 + flopy/mf6/utils/createpackages.py | 24 ++++++--- flopy/mf6/utils/templates/context.py.jinja | 62 +++++++++++++++++++++- 3 files changed, 79 insertions(+), 9 deletions(-) diff --git a/flopy/mf6/data/dfn/utl-tas.dfn b/flopy/mf6/data/dfn/utl-tas.dfn index 59d47f1422..81c6fd25bc 100644 --- a/flopy/mf6/data/dfn/utl-tas.dfn +++ b/flopy/mf6/data/dfn/utl-tas.dfn @@ -19,6 +19,7 @@ type keyword shape reader urword optional false +in_record true longname description xxx @@ -29,6 +30,7 @@ shape any1d tagged false reader urword optional false +in_record true longname description Name by which a package references a particular time-array series. The name must be unique among all time-array series used in a package. diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index bfa1468677..ccf439a192 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -489,9 +489,16 @@ def from_type(cls, t: type) -> Optional["VarKind"]: args = get_args(t) if origin is Union: if len(args) >= 2 and args[-1] is type(None): + if len(args) > 2: + return VarKind.Union return cls.from_type(args[0]) return VarKind.Union - if t is np.ndarray or origin is NDArray or origin is ArrayLike: + if ( + t is np.ndarray + or origin is np.ndarray + or origin is NDArray + or origin is ArrayLike + ): return VarKind.Array elif origin is collections.abc.Iterable or origin is list: return VarKind.List @@ -602,11 +609,12 @@ class Context: name: ContextName base: Optional[type] - parent: Optional[type] + parent: Optional[Union[type, str]] description: Optional[str] metadata: Metadata variables: Vars records: Vars + subpkg: bool _SCALAR_TYPES = { @@ -647,6 +655,7 @@ def make_context( common = common or dict() subpkgs = subpkgs or dict() + _subpkg = Subpkg.from_dfn(dfn) records = dict() def _nt_name(s): @@ -674,11 +683,11 @@ def _parent() -> Optional[str]: return "MFSimulation" if r == "nam": return "MFModel" - subpkg = Subpkg.from_dfn(dfn) - if subpkg: - if len(subpkg.parents) > 1: - return f"Union[{', '.join([_try_get_type_name(t) for t in subpkg.parents])}]" - return subpkg.parents[0] + + if _subpkg: + if len(_subpkg.parents) > 1: + return f"Union[{', '.join([_try_get_type_name(t) for t in _subpkg.parents])}]" + return _subpkg.parents[0] return "MFPackage" parent = _parent() @@ -1490,6 +1499,7 @@ def _to_dfn_fmt(var: Var) -> List[str]: metadata=_metadata(), variables=_variables(), records=records, + subpkg=bool(_subpkg), ) diff --git a/flopy/mf6/utils/templates/context.py.jinja b/flopy/mf6/utils/templates/context.py.jinja index f49e2270fd..6347c790cf 100644 --- a/flopy/mf6/utils/templates/context.py.jinja +++ b/flopy/mf6/utils/templates/context.py.jinja @@ -6,7 +6,7 @@ from typing import Any, Optional, Tuple, List, Dict, Union, Literal, Iterable from numpy.typing import NDArray from flopy.mf6.data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenerator -from flopy.mf6.mfpackage import MFPackage +from flopy.mf6.mfpackage import MFPackage, MFChildPackages from flopy.mf6.mfmodel import MFModel from flopy.mf6.mfsimbase import MFSimulationBase {% if base != "MFSimulationBase" %} @@ -22,4 +22,62 @@ class {% if base == "MFSimulationBase" %}MF{% else %}Modflow{% endif %}{{ name.t {% include "init.jinja" %} - {% include "load.jinja" %} \ No newline at end of file + {% include "load.jinja" %} + +{% if subpkg %} +class {{ name.title.title() }}Packages(MFChildPackages): + """ + {{ name.title.title() }}Packages is a container class for the Modflow{{ name.title.title() }} class. + + Methods + ------- + initialize + Initializes a new Modflow{{ name.title.title() }} package removing any sibling child + packages attached to the same parent package. See Modflow{{ name.title.title() }} init + documentation for definition of parameters. + append_package + Adds a new Modflow{{ name.title.title() }} package to the container. See Modflow{{ name.title.title() }} + init documentation for definition of parameters. + """ + + package_abbr = "{{ name.title.lower() }}packages" + + def initialize( + self, + {%- for name, var in variables.items() if ("parent" not in name and "loading" not in name) %} + {%- if var.default is defined %} + {{ name }}: {{ var._type }} = {{ var.default }}, + {%- else -%} + {{ name }}: {{ var._type }}, + {% endif -%} + {%- endfor %} + ): + new_package = Modflow{{ name.title.title() }}( + self._cpparent, + {%- for name, var in variables.items() if ("parent" not in name and "loading" not in name) %} + {{ name }}={{ name }}, + {%- endfor %} + child_builder_call=True, + ) + self.init_package(new_package, filename) + + def append_package( + self, + {%- for name, var in variables.items() if ("parent" not in name and "loading" not in name) %} + {%- if var.default is defined %} + {{ name }}: {{ var._type }} = {{ var.default }}, + {%- else -%} + {{ name }}: {{ var._type }}, + {% endif -%} + {%- endfor %} + ): + new_package = Modflow{{ name.title.title() }}( + self._cpparent, + {%- for name, var in variables.items() if ("parent" not in name and "loading" not in name ) %} + {{ name }}={{ name }}, + {%- endfor %} + pname=pname, + child_builder_call=True, + ) + self._append_package(new_package, filename) +{% endif %} \ No newline at end of file From 15f21dc45fef6aa23ac156a8a2bd55a211af4b59 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 2 Oct 2024 10:32:02 -0400 Subject: [PATCH 23/46] write __init__.py too --- flopy/mf6/utils/createpackages.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index ccf439a192..83f0cdd076 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -1580,7 +1580,7 @@ def make_all(dfndir: Path, outdir: Path, verbose: bool = False): subpkgs[subpkg.key] = subpkg # generate target files - for name, dfn in dfns.items(): + for dfn in dfns.values(): with open(p) as f: make_targets( dfn=dfn, @@ -1590,6 +1590,16 @@ def make_all(dfndir: Path, outdir: Path, verbose: bool = False): verbose=verbose, ) + # write __init__.py file + init_path = outdir / "__init__.py" + with open(init_path, "w") as f: + for dfn in dfns.values(): + prefix = "MF" if dfn.name == ("sim", "nam") else "Modflow" + context = dfn.name.contexts[0] + f.write( + f"from .mf{context.title} import {prefix}{context.title.title()}\n" + ) + # format the generated files run_cmd("ruff", "format", outdir, verbose=verbose) run_cmd("ruff", "check", "--fix", outdir, verbose=True) From 5f72225b0ad41e5b447bb26f1e1cbdbb2f74bb1a Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 2 Oct 2024 11:11:52 -0400 Subject: [PATCH 24/46] handle mfnam pkg correctly --- flopy/mf6/utils/createpackages.py | 33 +++++++++++++++++----------- flopy/mf6/utils/templates/init.jinja | 10 +++++---- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index 83f0cdd076..e443815336 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -268,6 +268,8 @@ def title(self) -> str: l, r = self if self == ("sim", "nam"): return "simulation" + if l is None: + return r if r is None: return l if l in ["sln", "exg"]: @@ -320,11 +322,17 @@ class DfnName(NamedTuple): @property def contexts(self) -> List[ContextName]: - if self.l != "sim" and self.r == "nam": - return [ - ContextName(*self), # nam pkg - ContextName(self.l, None), # model - ] + if self.r == "nam": + if self.l == "sim": + return [ + ContextName(None, self.r), # nam pkg + ContextName(*self), # simulation + ] + else: + return [ + ContextName(*self), # nam pkg + ContextName(self.l, None), # model + ] return [ContextName(*self)] @@ -660,10 +668,9 @@ def make_context( def _nt_name(s): """Trim the name of a record for a corresponding named tuple.""" - s = s.title().replace("record", "").replace("-", "_").replace("_", "") - # if s.endswith("s"): - # return s[:-1] - return s + return ( + s.title().replace("record", "").replace("-", "_").replace("_", "") + ) def _parent() -> Optional[str]: """ @@ -677,13 +684,12 @@ def _parent() -> Optional[str]: the need to import `MFSimulation/MFModel/MFPackage`. """ l, r = dfn.name - if (l, r) == ("sim", "nam"): + if (l, r) == ("sim", "nam") and name == ("sim", "nam"): return None if l in ["sim", "exg", "sln"]: return "MFSimulation" if r == "nam": return "MFModel" - if _subpkg: if len(_subpkg.parents) > 1: return f"Union[{', '.join([_try_get_type_name(t) for t in _subpkg.parents])}]" @@ -1141,7 +1147,8 @@ def _add_exg_vars(_vars: Vars) -> Vars: def _add_pkg_vars(_vars: Vars) -> Vars: """Add variables for a package context.""" - parent_name = "parent_" + ( + parent_prefix = "" if dfn.name == ("sim", "nam") else "parent_" + parent_name = parent_prefix + ( parent.lower() .replace("mf", "") .replace("union[", "") @@ -1219,7 +1226,7 @@ def _add_pkg_vars(_vars: Vars) -> Vars: # impossible if the data variable # doesn't appear in the reference # definition, though. - if subpkgs: + if subpkgs and name != (None, "nam"): for k, subpkg in subpkgs.items(): key = vars_.get(k, None) if not key: diff --git a/flopy/mf6/utils/templates/init.jinja b/flopy/mf6/utils/templates/init.jinja index 90c166c07a..83ce9b9519 100644 --- a/flopy/mf6/utils/templates/init.jinja +++ b/flopy/mf6/utils/templates/init.jinja @@ -62,20 +62,22 @@ def __init__( {% if name.l == "exg" -%} simulation.register_exchange_file(self) {%- endif %} - {% for name, var in variables.items() -%} + {% for n, var in variables.items() -%} {%- if var.init_assign -%} - self.{{ name }} = {{ name }} + self.{{ n }} = {{ n }} {% endif -%} {%- if var.subpkg is defined -%} - self._{{ name }} = self.build_mfdata("{{ name }}", None) + self._{{ n }} = self.build_mfdata("{{ n }}", None) + {%- if name.r != "nam" %} self._{{ var.subpkg.abbr }}_package = self.build_child_package( "{{ var.subpkg.abbr }}", {{ var.subpkg.val }}, "{{ var.subpkg.param }}", self.{{ var.subpkg.key }} ) + {%- endif %} {% elif var.init_build -%} - self.{{ name }} = self.build_mfdata("{{ name }}", {{ name }}) + self.{{ n }} = self.build_mfdata("{{ n }}", {{ n }}) {% endif -%} {%- endfor -%} self._init_complete = True From 576d989e4b9d81357e5ff9c4af8b39d19c972c1f Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 2 Oct 2024 11:27:45 -0400 Subject: [PATCH 25/46] minor fixes --- flopy/mf6/utils/createpackages.py | 2 +- flopy/mf6/utils/templates/attrs.jinja | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index e443815336..ba2cc6a548 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -122,7 +122,7 @@ def _try_get_type_name(t) -> str: elif origin is Union: if len(args) >= 2 and args[-1] is type(None): if len(args) > 2: - return f"Optional[Tuple[{', '.join([_try_get_type_name(a) for a in args[:-1]])}]]" + return f"Optional[Union[{', '.join([_try_get_type_name(a) for a in args[:-1]])}]]" return f"Optional[{_try_get_type_name(args[0])}]" return f"Union[{', '.join([_try_get_type_name(a) for a in args])}]" elif origin is tuple: diff --git a/flopy/mf6/utils/templates/attrs.jinja b/flopy/mf6/utils/templates/attrs.jinja index 30c774905b..d11780c983 100644 --- a/flopy/mf6/utils/templates/attrs.jinja +++ b/flopy/mf6/utils/templates/attrs.jinja @@ -1,4 +1,4 @@ - {%- if base != "MFSimulationBase" %} + {%- if base == "MFPackage" %} {% for var in variables.values() if var.class_attr %} {%- if var.kind == "list" or var.kind == "record" or var.kind == "union" %} {{ var.name }} = ListTemplateGenerator(("{{ name.l }}6", "{{ name.r }}", "{{ var.block }}", "{{ var.name }}")) From b6a468d7849bfe1374f6f43db133cd2412312597 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 2 Oct 2024 12:25:58 -0400 Subject: [PATCH 26/46] cleanup --- flopy/mf6/utils/createpackages.py | 135 +++++++++++++++++------------- 1 file changed, 76 insertions(+), 59 deletions(-) diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index ba2cc6a548..144a7bbee6 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -168,19 +168,14 @@ def renderable( based access but no arbitrary expressions, and only a limited set of custom filters. This can make it awkward to express some - conditionals. Conversely, Python classes - have some limitations (like no attribute - name collisions with reserved keywords). - So we convert the dataclasses we want to + things, so convert the dataclasses we'll pass to `template.render(...)` to dicts, with a few touchups. These include: - - removing key/value pairs whose value is None - - converting types to their suitably qualified names - - removing leading underscores (due to reserved keyword - collisions) from attribute names - - quoting strings forming the RHS of an assignment or + - converting types to suitably qualified type names + - optionally removing key/value pairs whose value is None + - optionally quoting strings forming the RHS of an assignment or argument passing expression """ @@ -196,7 +191,7 @@ def _render_key(k): def _render_val(v): return _try_get_type_name(_try_get_enum_value(v)) - # drop nones except for any explicitly kept + # drop nones except where keep requested _d = { _render_key(k): _render_val(v) for k, v in d.items() @@ -220,7 +215,6 @@ def render(self) -> dict: asdict(self, dict_factory=lambda d: _render(dict(d))) ) - setattr(cls, "_render", _render) setattr(cls, "render", render) return cls @@ -245,11 +239,10 @@ class ContextName(NamedTuple): From the `ContextName` several other things are derived, including: + - the input context class' name + - a description of the context class - the name of the source file to write - - the name of the input context class - - a description for the context class - - an optional base class for the context to inherit from - - an optional parent class which can own context instances + - the base class the context inherits from """ @@ -277,8 +270,8 @@ def title(self) -> str: return f"{l}{r}" @property - def base(self) -> Optional[str]: - """A base class from which the input context should inherit.""" + def base(self) -> str: + """Base class from which the input context should inherit.""" l, r = self if self == ("sim", "nam"): return "MFSimulationBase" @@ -315,6 +308,12 @@ class DfnName(NamedTuple): Uniquely identifies an input definition by its name, which consists of a <= 3-letter left term and an optional right term, also <= 3 letters. + + Notes + ----- + A single `DefinitionName` may be associated with one or + more `ContextName`s. For instance, a model DFN file will + produce both a NAM package class and also a model class. """ l: str @@ -322,6 +321,16 @@ class DfnName(NamedTuple): @property def contexts(self) -> List[ContextName]: + """ + Returns a list of contexts this definition will produce. + + Notes + ----- + Model definition files produce both a model class context and + a model namefile package context. The same goes for simulation + definition files. All other definition files produce a single + context. + """ if self.r == "nam": if self.l == "sim": return [ @@ -342,7 +351,7 @@ def contexts(self) -> List[ContextName]: @dataclass class Dfn(UserDict): """ - An MF6 input definition. Contains variables and metadata. + An MF6 input definition. """ name: Optional[DfnName] @@ -501,12 +510,7 @@ def from_type(cls, t: type) -> Optional["VarKind"]: return VarKind.Union return cls.from_type(args[0]) return VarKind.Union - if ( - t is np.ndarray - or origin is np.ndarray - or origin is NDArray - or origin is ArrayLike - ): + if origin is np.ndarray or origin is NDArray or origin is ArrayLike: return VarKind.Array elif origin is collections.abc.Iterable or origin is list: return VarKind.List @@ -568,6 +572,11 @@ def __init__( self.children = children self.meta = meta self.subpkg = subpkg + # TODO: the rest of the attributes below are + # needed to handle complexities in the input + # context classes; in a future version, they + # will ideally not be necessary. + # --- # the variable's general kind. # this is ofc derivable on demand but Jinja # doesn't allow arbitrary expressions, and it @@ -606,8 +615,14 @@ class Context: than one input context (e.g. model DFNs yield a model class and a package class). + Notes + ----- A context class minimally consists of a name, a map of variables, - and a list of metadata. + a map of records, and a list of metadata. + + A separate map of record variables is maintained because we will + generate named tuples for record types, and complex filtering of + e.g. nested maps of variables is awkward or impossible in Jinja. The context class may inherit from a base class, and may specify a parent context within which it can be created (the parent then @@ -667,7 +682,15 @@ def make_context( records = dict() def _nt_name(s): - """Trim the name of a record for a corresponding named tuple.""" + """ + Convert a record name to the name of a corresponding named tuple. + + Notes + ----- + Dashes and underscores are removed, with title-casing for clauses + separated by them, and a trailing "record" is removed if present. + + """ return ( s.title().replace("record", "").replace("-", "_").replace("_", "") ) @@ -680,8 +703,11 @@ def _parent() -> Optional[str]: will be a Union of possible parent types, otherwise a single parent type. + Notes + ----- We return a string directly instead of a type to avoid - the need to import `MFSimulation/MFModel/MFPackage`. + the need to import `MFSimulation` in this file (avoids + potential for circular imports). """ l, r = dfn.name if (l, r) == ("sim", "nam") and name == ("sim", "nam"): @@ -707,22 +733,22 @@ def _convert( an input definition to a specification suitable for type hints, docstrings, an `__init__` method's signature, etc. - This involves expanding nested input hierarchies, mapping + This involves expanding nested type hierarchies, mapping types to roughly equivalent Python primitives/composites, and other shaping. Notes ----- + The rules for optional variable defaults are as follows: If a `default_value` is not provided, keywords are `False` - by default. Everything else is `None` by default. + by default, everything else is `None`. If `wrap` is true, scalars will be wrapped as records with keywords represented as string literals. This is useful for unions, to distinguish between choices having the same type. - A map of subpackage references `refs` may be provided, in - which case any variable whose name is a key for any given - subpackage will detect and store the subpackage reference. + Any variable whose name functions as a key for a subpackage + will be provided with a subpackage reference. """ # var attributes to be converted @@ -762,7 +788,7 @@ def _description(descr: str) -> str: def _fields(record_name: str) -> Vars: """ - Load a record's fields and recursively convert them. + Recursively load/convert a record's fields. Notes ----- @@ -863,11 +889,6 @@ def _is_implicit_record(): records[_nt_name(record_name)] = replace( record, name=_nt_name(record_name) ) - # TODO: do we want to use named tuples here? - # it's a bit less explicit and requires looking - # back and forth between the tuple definition - # and the class docstring... but nice to have - # a concise definition of each record type... record_type = namedtuple( _nt_name(record_name), [_nt_name(k) for k in record_fields.keys()], @@ -886,7 +907,6 @@ def _is_implicit_record(): Union[tuple([c._type for c in children.values()])] ] - # basic composite types... # union (product), children are record choices elif _type.startswith("keystring"): names = _type.split()[1:] @@ -908,9 +928,9 @@ def _is_implicit_record(): type_ = record_type is_record = True - # are we wrapping a record which is a - # choice within a union? if so, use a - # literal for the keyword as tag e.g. + # are we wrapping a var into a record + # as a choice in a union? if so use a + # string literal for the keyword e.g. # `Tuple[Literal[...], T]` elif wrap: field_name = _name @@ -947,9 +967,9 @@ def _is_implicit_record(): # finally a bog standard scalar else: - # if it's a keyword, there are two cases we want to convert it to - # a string literal: if it's 1) tagging another variable, or 2) it - # is being wrapped into a record to represent a choice in a union + # if it's a keyword, there are 2 cases where we want to convert + # it to a string literal: 1) it tags another variable, or 2) it + # is being wrapped into a record as a choice in a union tag = _type == "keyword" and (tagged or wrap) type_ = Literal[_name] if tag else _SCALAR_TYPES.get(_type, _type) @@ -959,7 +979,8 @@ def _is_implicit_record(): # keywords default to False, everything else to None default = var.get("default", False if type_ is bool else None) - # if name is a reserved keyword, add a trailing underscore to it + # if name is a reserved keyword, add a trailing underscore to it. + # convert dashes to underscores since it may become a class attr. name_ = (f"{_name}_" if _name in kwlist else _name).replace("-", "_") # create var @@ -980,14 +1001,9 @@ def _is_implicit_record(): var_.init_build = False var_.subpkg = subpkg - # make named tuples for record types + # if this is a record, make a named tuple for it if is_record: records[_nt_name(name_)] = replace(var_, name=_nt_name(name_)) - # TODO: do we want to use named tuples here? - # it's a bit less explicit and requires looking - # back and forth between the tuple definition - # and the class docstring... but nice to have - # a concise definition of each record type... if children: type_ = namedtuple( _nt_name(name_), [_nt_name(k) for k in children.keys()] @@ -1006,15 +1022,15 @@ def _is_implicit_record(): def _variables() -> Vars: """ - Convert the input variables to parameters for an input - context class. Context-specific parameters may also be - added depending. + Return all input variables for an input context class. Notes ----- Not all variables become parameters; nested variables will become components of composite parameters, e.g., record fields, keystring (union) choices, list items. + + Variables may be added, depending on the context type. """ vars_ = dfn.copy() @@ -1477,7 +1493,7 @@ def _add_sim_params(_vars: Vars) -> Vars: def _metadata() -> List[Metadata]: """ - Get a list of the class' original definition attributes, + Get a list of the class' original definition attributes as a partial, internal reproduction of the DFN contents. Notes @@ -1488,14 +1504,15 @@ def _metadata() -> List[Metadata]: Python, consolidating nested types, etc. """ - def _to_dfn_fmt(var: Var) -> List[str]: + def _fmt_var(var: Var) -> List[str]: exclude = ["longname", "description"] return [ " ".join([k, v]) for k, v in var.items() if k not in exclude ] - return [["header"] + [attr for attr in (dfn.metadata or list())]] + [ - _to_dfn_fmt(var) for var in dfn.values() + meta = dfn.metadata or list() + return [["header"] + [m for m in meta]] + [ + _fmt_var(var) for var in dfn.values() ] return Context( From ac6842fb2d0df94f41e0a5ba7589d8af7df31fd4 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 2 Oct 2024 13:47:41 -0400 Subject: [PATCH 27/46] cleanup/fixes --- flopy/mf6/utils/createpackages.py | 11 +++++------ .../mf6/utils/templates/docstring_params.jinja | 14 +++++++------- flopy/mf6/utils/templates/init.jinja | 4 +--- flopy/mf6/utils/templates/records.jinja | 17 ++++++++++++++++- 4 files changed, 29 insertions(+), 17 deletions(-) diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index 144a7bbee6..f86fb16ae6 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -265,6 +265,8 @@ def title(self) -> str: return r if r is None: return l + if l == "sim": + return r if l in ["sln", "exg"]: return r return f"{l}{r}" @@ -296,7 +298,7 @@ def description(self) -> str: ) elif self.base == "MFModel": return f"Modflow{title} defines a {l.upper()} model." - elif self.base == "MFSimulation": + elif self.base == "MFSimulationBase": return """ MFSimulation is used to load, build, and/or save a MODFLOW 6 simulation. A MFSimulation object must be created before creating any of the MODFLOW 6 @@ -885,6 +887,7 @@ def _is_implicit_record(): _type=record_type, block=block, children=record_fields, + description=description, ) records[_nt_name(record_name)] = replace( record, name=_nt_name(record_name) @@ -1175,11 +1178,7 @@ def _add_pkg_vars(_vars: Vars) -> Vars: parent_name: Var( name=parent_name, _type=parent, - description=( - f"{parent_name.title()} that this package is part of. " - f"Package is automatically added to the {parent_name} " - "when it is initialized." - ), + description="Parent that this package is part of.", init_param=True, init_assign=False, init_super=True, diff --git a/flopy/mf6/utils/templates/docstring_params.jinja b/flopy/mf6/utils/templates/docstring_params.jinja index 56bb8750bf..184dff93ab 100644 --- a/flopy/mf6/utils/templates/docstring_params.jinja +++ b/flopy/mf6/utils/templates/docstring_params.jinja @@ -1,9 +1,9 @@ -{% for var in variables.values() recursive %} - {% if loop.depth > 1 %}* {% endif %}{{ var.name }} : {{ var._type }} -{%- if var.description is defined and not var.is_choice %} -{{ var.description|wordwrap|indent(4 + (loop.depth * 4), first=True) }} +{%- for v in variables.values() recursive %} + * {{ v.name }} : {{ v._type }} +{%- if v.description is defined and not v.is_choice %} +{{ v.description|wordwrap|indent(4 + (loop.depth * 4), first=True) }} {%- endif %} -{%- if var.children is defined -%} -{{ loop(var.children.values())|indent(4) }} +{%- if v.children is defined and loop.depth < 2 -%} +{{ loop(v.children.values())|indent(4) }} {%- endif %} -{% endfor %} \ No newline at end of file +{% endfor -%} \ No newline at end of file diff --git a/flopy/mf6/utils/templates/init.jinja b/flopy/mf6/utils/templates/init.jinja index 83ce9b9519..91c3c72600 100644 --- a/flopy/mf6/utils/templates/init.jinja +++ b/flopy/mf6/utils/templates/init.jinja @@ -44,9 +44,7 @@ def __init__( {%- endfor %} {% elif base == "MFPackage" %} super().__init__( - {% if parent == "MFSimulation" -%} - simulation, - {% elif parent == "MFModel" -%} + {% if parent == "MFModel" -%} model, {% endif -%} package_type="{{ name.r }}", diff --git a/flopy/mf6/utils/templates/records.jinja b/flopy/mf6/utils/templates/records.jinja index a12f855cbe..cb67bcefe6 100644 --- a/flopy/mf6/utils/templates/records.jinja +++ b/flopy/mf6/utils/templates/records.jinja @@ -1,3 +1,18 @@ {%- for name, var in records.items() -%} {{ name }} = {{ var._type }} -{% endfor -%} \ No newline at end of file +""" +{%- if var.description is defined %} +{{ var.description|wordwrap }} +{%- endif %} +{%- for v in var.children.values() recursive %} + * {{ v.name }} : {{ v._type }} +{%- if v.description is defined and not v.is_choice %} +{{ v.description|wordwrap|indent(4 + (loop.depth * 4), first=True) }} +{%- endif %} +{%- if v.children is defined -%} +{{ loop(v.children.values())|indent(4) }} +{%- endif %} +{% endfor -%} +""" + +{% endfor -%} From 90de4c2ab6e23ff506a9b648e43a5c77b8731059 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 2 Oct 2024 13:51:45 -0400 Subject: [PATCH 28/46] only import MFSimulationBase if needed (avoid circularity) --- flopy/mf6/utils/templates/context.py.jinja | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flopy/mf6/utils/templates/context.py.jinja b/flopy/mf6/utils/templates/context.py.jinja index 6347c790cf..be2f76f845 100644 --- a/flopy/mf6/utils/templates/context.py.jinja +++ b/flopy/mf6/utils/templates/context.py.jinja @@ -8,8 +8,9 @@ from numpy.typing import NDArray from flopy.mf6.data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenerator from flopy.mf6.mfpackage import MFPackage, MFChildPackages from flopy.mf6.mfmodel import MFModel +{% if base == "MFSimulationBase" %} from flopy.mf6.mfsimbase import MFSimulationBase -{% if base != "MFSimulationBase" %} +{% else %} from flopy.mf6.modflow.mfsimulation import MFSimulation {% endif %} From 3d59da9eda1b5f80baa61b0d7b644600f5f2f751 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 2 Oct 2024 15:56:35 -0400 Subject: [PATCH 29/46] fix lists of unions/tuples --- flopy/mf6/mfmodel.py | 3 + flopy/mf6/utils/createpackages.py | 80 ++++++++++++++-------- flopy/mf6/utils/templates/attrs.jinja | 8 +-- flopy/mf6/utils/templates/context.py.jinja | 20 +++--- flopy/mf6/utils/templates/init.jinja | 47 ++++++------- 5 files changed, 90 insertions(+), 68 deletions(-) diff --git a/flopy/mf6/mfmodel.py b/flopy/mf6/mfmodel.py index 3a54e4f525..bcccbe087b 100644 --- a/flopy/mf6/mfmodel.py +++ b/flopy/mf6/mfmodel.py @@ -81,8 +81,11 @@ def __init__( structure=None, model_rel_path=".", verbose=False, + parent=None, **kwargs, ): + if parent: + simulation = parent super().__init__(simulation.simulation_data, modelname) self.simulation = simulation self.simulation_data = simulation.simulation_data diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index f86fb16ae6..0e7c722161 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -292,10 +292,7 @@ def description(self) -> str: l, r = self title = self.title.title() if self.base == "MFPackage": - return ( - f"Modflow{title} defines a {r} package within a {l} " - f"{self.base.lower().replace('mf', '')}." - ) + return f"Modflow{title} defines a {r.upper()} package." elif self.base == "MFModel": return f"Modflow{title} defines a {l.upper()} model." elif self.base == "MFSimulationBase": @@ -862,7 +859,9 @@ def _fields(record_name: str) -> Vars: "type" ].startswith("record") - def _is_implicit_record(): + def _is_implicit_scalar_record(): + # if the record is defined implicitly and it has + # only scalar fields types = [ _try_get_type_name(v["type"]) for n, v in dfn.items() @@ -877,7 +876,7 @@ def _is_implicit_record(): record_type = _convert(record_spec, wrap=False) children = {_nt_name(record_name).lower(): record_type} type_ = Iterable[record_type._type] - elif _is_implicit_record(): + elif _is_implicit_scalar_record(): record_name = _name record_fields = _fields(record_name) field_types = [f._type for f in record_fields.values()] @@ -904,11 +903,39 @@ def _is_implicit_record(): children = {_nt_name(record_name): record} type_ = Iterable[record_type] else: - # irregular recarray, rows can be any of several types - children = {n: _convert(dfn[n], wrap=False) for n in names} - type_ = Iterable[ - Union[tuple([c._type for c in children.values()])] - ] + # implicit complex record (i.e. some fields are records or unions) + record_fields = { + n: _convert(dfn[n], wrap=False) for n in names + } + first = list(record_fields.values())[0] + single = len(record_fields) == 1 + record_name = first.name if single else _name + _t = [f._type for f in record_fields.values()] + record_type = ( + first._type + if (single and first.kind == VarKind.Union) + else Tuple[tuple(_t)] + ) + record = Var( + name=record_name, + _type=record_type, + block=block, + children=first.children if single else record_fields, + description=description, + ) + records[_nt_name(record_name)] = replace( + record, name=_nt_name(record_name) + ) + record_type = namedtuple( + _nt_name(record_name), + [_nt_name(k) for k in record_fields.keys()], + ) + record = replace( + record, + _type=record_type, + name=_nt_name(record_name).lower(), + ) + type_ = Iterable[record_type] # union (product), children are record choices elif _type.startswith("keystring"): @@ -981,6 +1008,8 @@ def _is_implicit_record(): # keywords default to False, everything else to None default = var.get("default", False if type_ is bool else None) + if _name in ["continue", "print_input"]: # hack... + default = None # if name is a reserved keyword, add a trailing underscore to it. # convert dashes to underscores since it may become a class attr. @@ -1081,6 +1110,7 @@ def _add_exg_vars(_vars: Vars) -> Vars: "Do not set this parameter. It is intended for " "debugging and internal processing purposes only." ), + default=False, init_param=True, init_assign=False, init_build=False, @@ -1151,7 +1181,7 @@ def _add_exg_vars(_vars: Vars) -> Vars: continue vars_[subpkg.key].init_param = False vars_[subpkg.key].class_attr = True - vars_[subpkg.key].subpkg = None + # vars_[subpkg.key].subpkg = None vars_[subpkg.val] = Var( name=subpkg.val, description=subpkg.description, @@ -1166,14 +1196,7 @@ def _add_exg_vars(_vars: Vars) -> Vars: def _add_pkg_vars(_vars: Vars) -> Vars: """Add variables for a package context.""" - parent_prefix = "" if dfn.name == ("sim", "nam") else "parent_" - parent_name = parent_prefix + ( - parent.lower() - .replace("mf", "") - .replace("union[", "") - .replace("]", "") - .replace(", ", "_or_") - ) + parent_name = "parent" vars_ = { parent_name: Var( name=parent_name, @@ -1190,6 +1213,7 @@ def _add_pkg_vars(_vars: Vars) -> Vars: "Do not set this variable. It is intended for debugging " "and internal processing purposes only." ), + default=False, init_param=True, init_assign=False, init_super=True, @@ -1247,8 +1271,8 @@ def _add_pkg_vars(_vars: Vars) -> Vars: if not key: continue vars_[subpkg.key].init_param = False + vars_[subpkg.key].init_build = True vars_[subpkg.key].class_attr = True - vars_[subpkg.key].subpkg = None vars_[subpkg.val] = Var( name=subpkg.val, description=subpkg.description, @@ -1348,7 +1372,6 @@ def _add_mdl_vars(_vars: Vars) -> Vars: continue vars_[subpkg.key].init_param = False vars_[subpkg.key].class_attr = True - vars_[subpkg.key].subpkg = None vars_[subpkg.val] = Var( name=subpkg.val, description=subpkg.description, @@ -1463,7 +1486,6 @@ def _add_sim_params(_vars: Vars) -> Vars: vars_[subpkg.key].init_param = False vars_[subpkg.key].init_build = False vars_[subpkg.key].class_attr = True - vars_[subpkg.key].subpkg = None vars_[subpkg.param] = Var( name=subpkg.param, description=subpkg.description, @@ -1617,11 +1639,13 @@ def make_all(dfndir: Path, outdir: Path, verbose: bool = False): init_path = outdir / "__init__.py" with open(init_path, "w") as f: for dfn in dfns.values(): - prefix = "MF" if dfn.name == ("sim", "nam") else "Modflow" - context = dfn.name.contexts[0] - f.write( - f"from .mf{context.title} import {prefix}{context.title.title()}\n" - ) + for context in dfn.name.contexts: + prefix = ( + "MF" if context.base == "MFSimulationBase" else "Modflow" + ) + f.write( + f"from .mf{context.title} import {prefix}{context.title.title()}\n" + ) # format the generated files run_cmd("ruff", "format", outdir, verbose=verbose) diff --git a/flopy/mf6/utils/templates/attrs.jinja b/flopy/mf6/utils/templates/attrs.jinja index d11780c983..349c400bfd 100644 --- a/flopy/mf6/utils/templates/attrs.jinja +++ b/flopy/mf6/utils/templates/attrs.jinja @@ -1,17 +1,17 @@ {%- if base == "MFPackage" %} {% for var in variables.values() if var.class_attr %} {%- if var.kind == "list" or var.kind == "record" or var.kind == "union" %} - {{ var.name }} = ListTemplateGenerator(("{{ name.l }}6", "{{ name.r }}", "{{ var.block }}", "{{ var.name }}")) + {{ var.name }} = ListTemplateGenerator(({% if name.l is not none and name.l != "sim" %}"{{ name.l }}6", {% endif %}"{{ name.r }}", "{{ var.block }}", "{{ var.name }}")) {%- elif var.kind == "array" %} - {{ var.name }} = ArrayTemplateGenerator(("{{ name.l }}6", "{{ name.r }}", "{{ var.block }}", "{{ var.name }}")) + {{ var.name }} = ArrayTemplateGenerator(({% if name.l is not none and name.l != "sim" %}"{{ name.l }}6", {% endif %}"{{ name.r }}", "{{ var.block }}", "{{ var.name }}")) {%- endif -%} {%- endfor %} {% endif -%} {%- if base == "MFModel" %} model_type = "{{ name.title }}" {%- elif base == "MFPackage" %} - package_abbr = "{{ name.l }}{{ name.r }}" + package_abbr = "{% if name.l != "sln" and name.l != "sim" and name.l is not none %}{{ name.l }}{% endif %}{{ name.r }}" _package_type = "{{ name.r }}" - dfn_file_name = "{{ name.l }}-{{ name.r }}.dfn" + dfn_file_name = "{% if name.l != "sln" and name.l is not none %}{{ name.l }}-{% elif name.l is none %}sim-{% endif %}{{ name.r }}.dfn" dfn = {{ metadata|pprint|indent(10) }} {% endif -%} \ No newline at end of file diff --git a/flopy/mf6/utils/templates/context.py.jinja b/flopy/mf6/utils/templates/context.py.jinja index be2f76f845..b0402ddc53 100644 --- a/flopy/mf6/utils/templates/context.py.jinja +++ b/flopy/mf6/utils/templates/context.py.jinja @@ -10,8 +10,6 @@ from flopy.mf6.mfpackage import MFPackage, MFChildPackages from flopy.mf6.mfmodel import MFModel {% if base == "MFSimulationBase" %} from flopy.mf6.mfsimbase import MFSimulationBase -{% else %} -from flopy.mf6.modflow.mfsimulation import MFSimulation {% endif %} {% include "records.jinja" %} @@ -25,6 +23,7 @@ class {% if base == "MFSimulationBase" %}MF{% else %}Modflow{% endif %}{{ name.t {% include "load.jinja" %} +{# inlining all this below since it can ideally be made unnecessary before long? -#} {% if subpkg %} class {{ name.title.title() }}Packages(MFChildPackages): """ @@ -45,8 +44,10 @@ class {{ name.title.title() }}Packages(MFChildPackages): def initialize( self, - {%- for name, var in variables.items() if ("parent" not in name and "loading" not in name) %} - {%- if var.default is defined %} + {%- for name, var in variables.items() if ("simulation" not in name and "model" not in name and "package" not in name and "loading" not in name ) %} + {%- if var._type == "MFSimulation" %} + {{ name }} = None, + {%- elif var.default is defined %} {{ name }}: {{ var._type }} = {{ var.default }}, {%- else -%} {{ name }}: {{ var._type }}, @@ -55,7 +56,7 @@ class {{ name.title.title() }}Packages(MFChildPackages): ): new_package = Modflow{{ name.title.title() }}( self._cpparent, - {%- for name, var in variables.items() if ("parent" not in name and "loading" not in name) %} + {%- for name, var in variables.items() if ("simulation" not in name and "model" not in name and "package" not in name and "loading" not in name ) %} {{ name }}={{ name }}, {%- endfor %} child_builder_call=True, @@ -64,8 +65,10 @@ class {{ name.title.title() }}Packages(MFChildPackages): def append_package( self, - {%- for name, var in variables.items() if ("parent" not in name and "loading" not in name) %} - {%- if var.default is defined %} + {%- for name, var in variables.items() if ("simulation" not in name and "model" not in name and "package" not in name and "loading" not in name ) %} + {%- if var._type == "MFSimulation" %} + {{ name }} = None, + {%- elif var.default is defined %} {{ name }}: {{ var._type }} = {{ var.default }}, {%- else -%} {{ name }}: {{ var._type }}, @@ -74,10 +77,9 @@ class {{ name.title.title() }}Packages(MFChildPackages): ): new_package = Modflow{{ name.title.title() }}( self._cpparent, - {%- for name, var in variables.items() if ("parent" not in name and "loading" not in name ) %} + {%- for name, var in variables.items() if ("simulation" not in name and "model" not in name and "package" not in name and "loading" not in name) %} {{ name }}={{ name }}, {%- endfor %} - pname=pname, child_builder_call=True, ) self._append_package(new_package, filename) diff --git a/flopy/mf6/utils/templates/init.jinja b/flopy/mf6/utils/templates/init.jinja index 91c3c72600..a4a1ae4eca 100644 --- a/flopy/mf6/utils/templates/init.jinja +++ b/flopy/mf6/utils/templates/init.jinja @@ -1,7 +1,9 @@ def __init__( self, {%- for name, var in variables.items() if var.init_param %} - {%- if var.default is defined %} + {%- if var._type == "MFSimulation" %} + {{ name }} = None, + {%- elif var.default is defined %} {{ name }}: {{ var._type }} = {{ var.default }}, {%- else -%} {{ name }}: {{ var._type }}, @@ -11,15 +13,15 @@ def __init__( ): {% if base == "MFSimulationBase" %} super().__init__( - {%- for name, var in variables.items() if var.init_super %} - {{ name }}={{ name }}, + {%- for n, var in variables.items() if var.init_super %} + {{ n }}={{ n }}, {%- endfor %} **kwargs ) - {%- for name, var in variables.items() %} + {%- for n, var in variables.items() %} {%- if var.block == "options" and var.init_build %} - self.name_file.{{ name }}.set_data({{ name }}) - self.{{ name }} = self.name_file.{{ name }} + self.name_file.{{ n }}.set_data({{ n }}) + self.{{ n }} = self.name_file.{{ n }} {% endif -%} {%- if var.subpkg is defined %} self.{{ var.subpkg.param }} = self._create_package( @@ -30,31 +32,24 @@ def __init__( {% endfor -%} {% elif base == "MFModel" %} super().__init__( - model_type="{{ name.l }}6", - {%- for name, var in variables.items() if var.init_super %} - {{ name }}={{ name }}, + {%- for n, var in variables.items() if var.init_super %} + {{ n }}={{ n }}, {%- endfor %} + model_type="{{ name.l }}6", **kwargs, ) - {%- for name, var in variables.items() %} + {%- for n, var in variables.items() %} {%- if var.block == "options" and var.init_build %} - self.name_file.{{ name }}.set_data({{ name }}) - self.{{ name }} = self.name_file.{{ name }} + self.name_file.{{ n }}.set_data({{ n }}) + self.{{ n }} = self.name_file.{{ n }} {% endif -%} {%- endfor %} {% elif base == "MFPackage" %} super().__init__( - {% if parent == "MFModel" -%} - model, - {% endif -%} - package_type="{{ name.r }}", - {%- for name, var in variables.items() if var.init_super %} - {% if "parent" in name -%} - parent={{ name }}, - {%- else -%} - {{ name|replace("_package", "") }}={{ name }}, - {%- endif -%} + {%- for n, var in variables.items() if var.init_super %} + {{ n }}={{ n }}, {%- endfor %} + package_type="{{ name.r }}", **kwargs ) {% if name.l == "exg" -%} @@ -63,19 +58,17 @@ def __init__( {% for n, var in variables.items() -%} {%- if var.init_assign -%} self.{{ n }} = {{ n }} - {% endif -%} - {%- if var.subpkg is defined -%} + {% elif var.subpkg is defined and var.init_build -%} self._{{ n }} = self.build_mfdata("{{ n }}", None) - {%- if name.r != "nam" %} + {% elif var.subpkg is defined and name.r != "nam" -%} self._{{ var.subpkg.abbr }}_package = self.build_child_package( "{{ var.subpkg.abbr }}", {{ var.subpkg.val }}, "{{ var.subpkg.param }}", self.{{ var.subpkg.key }} ) - {%- endif %} {% elif var.init_build -%} - self.{{ n }} = self.build_mfdata("{{ n }}", {{ n }}) + self.{{ n }} = self.build_mfdata("{% if n == "continue_" %}continue{% else %}{{ n }}{% endif %}", {{ n }}) {% endif -%} {%- endfor -%} self._init_complete = True From d391220199c2c91a45ef3e84a4e8acbe40401a4b Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 2 Oct 2024 20:16:32 -0400 Subject: [PATCH 30/46] deduplicate 'auxiliary' var names --- flopy/mf6/data/dfn/gwe-lke.dfn | 4 ++-- flopy/mf6/data/dfn/gwe-mwe.dfn | 4 ++-- flopy/mf6/data/dfn/gwe-sfe.dfn | 4 ++-- flopy/mf6/data/dfn/gwe-uze.dfn | 4 ++-- flopy/mf6/data/dfn/gwf-lak.dfn | 4 ++-- flopy/mf6/data/dfn/gwf-maw.dfn | 4 ++-- flopy/mf6/data/dfn/gwf-sfr.dfn | 4 ++-- flopy/mf6/data/dfn/gwt-lkt.dfn | 4 ++-- flopy/mf6/data/dfn/gwt-mwt.dfn | 4 ++-- flopy/mf6/data/dfn/gwt-sft.dfn | 4 ++-- flopy/mf6/utils/createpackages.py | 14 +++++++++++--- flopy/mf6/utils/templates/attrs.jinja | 2 +- flopy/mf6/utils/templates/init.jinja | 2 +- 13 files changed, 33 insertions(+), 25 deletions(-) diff --git a/flopy/mf6/data/dfn/gwe-lke.dfn b/flopy/mf6/data/dfn/gwe-lke.dfn index b59b50420b..47b82a3fcc 100644 --- a/flopy/mf6/data/dfn/gwe-lke.dfn +++ b/flopy/mf6/data/dfn/gwe-lke.dfn @@ -442,7 +442,7 @@ description real or character value that defines the temperature of external inf block period name auxiliaryrecord -type record auxiliary auxname auxval +type record aux auxname auxval shape tagged in_record true @@ -451,7 +451,7 @@ longname description block period -name auxiliary +name aux type keyword shape in_record true diff --git a/flopy/mf6/data/dfn/gwe-mwe.dfn b/flopy/mf6/data/dfn/gwe-mwe.dfn index c805b6533f..c55b6f4324 100644 --- a/flopy/mf6/data/dfn/gwe-mwe.dfn +++ b/flopy/mf6/data/dfn/gwe-mwe.dfn @@ -408,7 +408,7 @@ description real or character value that defines the injection solute temperatur block period name auxiliaryrecord -type record auxiliary auxname auxval +type record aux auxname auxval shape tagged in_record true @@ -417,7 +417,7 @@ longname description block period -name auxiliary +name aux type keyword shape in_record true diff --git a/flopy/mf6/data/dfn/gwe-sfe.dfn b/flopy/mf6/data/dfn/gwe-sfe.dfn index 610e3911ff..d556241ff1 100644 --- a/flopy/mf6/data/dfn/gwe-sfe.dfn +++ b/flopy/mf6/data/dfn/gwe-sfe.dfn @@ -441,7 +441,7 @@ description real or character value that defines the temperature of inflow $(^{\ block period name auxiliaryrecord -type record auxiliary auxname auxval +type record aux auxname auxval shape tagged in_record true @@ -450,7 +450,7 @@ longname description block period -name auxiliary +name aux type keyword shape in_record true diff --git a/flopy/mf6/data/dfn/gwe-uze.dfn b/flopy/mf6/data/dfn/gwe-uze.dfn index 1f272617b7..10fda90e89 100644 --- a/flopy/mf6/data/dfn/gwe-uze.dfn +++ b/flopy/mf6/data/dfn/gwe-uze.dfn @@ -399,7 +399,7 @@ description real or character value that states what fraction of the simulated u block period name auxiliaryrecord -type record auxiliary auxname auxval +type record aux auxname auxval shape tagged in_record true @@ -408,7 +408,7 @@ longname description block period -name auxiliary +name aux type keyword shape in_record true diff --git a/flopy/mf6/data/dfn/gwf-lak.dfn b/flopy/mf6/data/dfn/gwf-lak.dfn index 3dc9e940c0..4dc7714adf 100644 --- a/flopy/mf6/data/dfn/gwf-lak.dfn +++ b/flopy/mf6/data/dfn/gwf-lak.dfn @@ -845,7 +845,7 @@ description real or character value that defines the bed slope for the lake outl block period name auxiliaryrecord -type record auxiliary auxname auxval +type record aux auxname auxval shape tagged in_record true @@ -854,7 +854,7 @@ longname description block period -name auxiliary +name aux type keyword shape in_record true diff --git a/flopy/mf6/data/dfn/gwf-maw.dfn b/flopy/mf6/data/dfn/gwf-maw.dfn index 2e957ec2c8..9bb9ddb812 100644 --- a/flopy/mf6/data/dfn/gwf-maw.dfn +++ b/flopy/mf6/data/dfn/gwf-maw.dfn @@ -723,7 +723,7 @@ description height above the pump elevation (SCALING\_LENGTH). If the simulated block period name auxiliaryrecord -type record auxiliary auxname auxval +type record aux auxname auxval shape tagged in_record true @@ -732,7 +732,7 @@ longname description block period -name auxiliary +name aux type keyword shape in_record true diff --git a/flopy/mf6/data/dfn/gwf-sfr.dfn b/flopy/mf6/data/dfn/gwf-sfr.dfn index eb3967e441..7b77b096a1 100644 --- a/flopy/mf6/data/dfn/gwf-sfr.dfn +++ b/flopy/mf6/data/dfn/gwf-sfr.dfn @@ -880,7 +880,7 @@ description character string that defines the path and filename for the file con block period name auxiliaryrecord -type record auxiliary auxname auxval +type record aux auxname auxval shape tagged in_record true @@ -889,7 +889,7 @@ longname description block period -name auxiliary +name aux type keyword shape in_record true diff --git a/flopy/mf6/data/dfn/gwt-lkt.dfn b/flopy/mf6/data/dfn/gwt-lkt.dfn index 6dbca6ffb1..484c0d1210 100644 --- a/flopy/mf6/data/dfn/gwt-lkt.dfn +++ b/flopy/mf6/data/dfn/gwt-lkt.dfn @@ -421,7 +421,7 @@ description real or character value that defines the concentration of external i block period name auxiliaryrecord -type record auxiliary auxname auxval +type record aux auxname auxval shape tagged in_record true @@ -430,7 +430,7 @@ longname description block period -name auxiliary +name aux type keyword shape in_record true diff --git a/flopy/mf6/data/dfn/gwt-mwt.dfn b/flopy/mf6/data/dfn/gwt-mwt.dfn index b2b4346785..05270b10c8 100644 --- a/flopy/mf6/data/dfn/gwt-mwt.dfn +++ b/flopy/mf6/data/dfn/gwt-mwt.dfn @@ -388,7 +388,7 @@ description real or character value that defines the injection solute concentrat block period name auxiliaryrecord -type record auxiliary auxname auxval +type record aux auxname auxval shape tagged in_record true @@ -397,7 +397,7 @@ longname description block period -name auxiliary +name aux type keyword shape in_record true diff --git a/flopy/mf6/data/dfn/gwt-sft.dfn b/flopy/mf6/data/dfn/gwt-sft.dfn index 5323f4c7c5..a26b4eb5ef 100644 --- a/flopy/mf6/data/dfn/gwt-sft.dfn +++ b/flopy/mf6/data/dfn/gwt-sft.dfn @@ -421,7 +421,7 @@ description real or character value that defines the concentration of inflow $(M block period name auxiliaryrecord -type record auxiliary auxname auxval +type record aux auxname auxval shape tagged in_record true @@ -430,7 +430,7 @@ longname description block period -name auxiliary +name aux type keyword shape in_record true diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index 0e7c722161..38a1198eb2 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -376,6 +376,7 @@ def load_dfn(f, name: Optional[DfnName] = None) -> Dfn: a dictionary of variable specifications as well as a list of metadata attributes. """ + meta = None vars_ = dict() var = dict() @@ -392,6 +393,13 @@ def load_dfn(f, name: Optional[DfnName] = None) -> Dfn: if meta is None: meta = list() meta.append(tail.strip()) + continue + head, sep, tail = line.partition("package-type") + if sep == "package-type": + if meta is None: + meta = list + meta.append(f"{sep} {tail.strip()}") + continue continue # if we hit a newline and the parameter dict @@ -1090,8 +1098,8 @@ def _add_exg_vars(_vars: Vars) -> Vars: b = name.r[:3] default = f"{a.upper()}6-{b.upper()}6" vars_ = { - "simulation": Var( - name="simulation", + "parent": Var( + name="parent", _type="MFSimulation", description=( "Simulation that this package is a part of. " @@ -1101,7 +1109,7 @@ def _add_exg_vars(_vars: Vars) -> Vars: init_param=True, init_assign=False, init_build=False, - init_super=False, + init_super=True, ), "loading_package": Var( name="loading_package", diff --git a/flopy/mf6/utils/templates/attrs.jinja b/flopy/mf6/utils/templates/attrs.jinja index 349c400bfd..bcc2c280d0 100644 --- a/flopy/mf6/utils/templates/attrs.jinja +++ b/flopy/mf6/utils/templates/attrs.jinja @@ -10,7 +10,7 @@ {%- if base == "MFModel" %} model_type = "{{ name.title }}" {%- elif base == "MFPackage" %} - package_abbr = "{% if name.l != "sln" and name.l != "sim" and name.l is not none %}{{ name.l }}{% endif %}{{ name.r }}" + package_abbr = "{% if name.l != "sln" and name.l != "sim" and name.l != "exg" and name.l is not none %}{{ name.l }}{% endif %}{{ name.r }}" _package_type = "{{ name.r }}" dfn_file_name = "{% if name.l != "sln" and name.l is not none %}{{ name.l }}-{% elif name.l is none %}sim-{% endif %}{{ name.r }}.dfn" dfn = {{ metadata|pprint|indent(10) }} diff --git a/flopy/mf6/utils/templates/init.jinja b/flopy/mf6/utils/templates/init.jinja index a4a1ae4eca..0833d1a404 100644 --- a/flopy/mf6/utils/templates/init.jinja +++ b/flopy/mf6/utils/templates/init.jinja @@ -53,7 +53,7 @@ def __init__( **kwargs ) {% if name.l == "exg" -%} - simulation.register_exchange_file(self) + parent.register_exchange_file(self) {%- endif %} {% for n, var in variables.items() -%} {%- if var.init_assign -%} From 201590703605e2fbae1595ed2c7c47c5b6b1a9bb Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 2 Oct 2024 20:40:08 -0400 Subject: [PATCH 31/46] fixes --- flopy/mf6/utils/createpackages.py | 29 +++++---- flopy/mf6/utils/templates/attrs.jinja | 6 +- flopy/mf6/utils/templates/context.py.jinja | 61 +------------------ .../utils/templates/docstring_params.jinja | 2 +- flopy/mf6/utils/templates/init.jinja | 6 +- flopy/mf6/utils/templates/packages.jinja | 58 ++++++++++++++++++ 6 files changed, 85 insertions(+), 77 deletions(-) create mode 100644 flopy/mf6/utils/templates/packages.jinja diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index 38a1198eb2..ec0114a44a 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -688,7 +688,7 @@ def make_context( _subpkg = Subpkg.from_dfn(dfn) records = dict() - def _nt_name(s): + def _nt_name(s, trims=False): """ Convert a record name to the name of a corresponding named tuple. @@ -698,9 +698,10 @@ def _nt_name(s): separated by them, and a trailing "record" is removed if present. """ - return ( - s.title().replace("record", "").replace("-", "_").replace("_", "") - ) + s = s.title().replace("record", "").replace("-", "_").replace("_", "") + if trims: + s = s[:-1] if s.endswith("s") else s + return s def _parent() -> Optional[str]: """ @@ -896,19 +897,19 @@ def _is_implicit_scalar_record(): children=record_fields, description=description, ) - records[_nt_name(record_name)] = replace( - record, name=_nt_name(record_name) + records[_nt_name(record_name, trims=True)] = replace( + record, name=_nt_name(record_name, trims=True) ) record_type = namedtuple( - _nt_name(record_name), + _nt_name(record_name, trims=True), [_nt_name(k) for k in record_fields.keys()], ) record = replace( record, _type=record_type, - name=_nt_name(record_name).lower(), + name=_nt_name(record_name, trims=True).lower(), ) - children = {_nt_name(record_name): record} + children = {_nt_name(record_name, trims=True): record} type_ = Iterable[record_type] else: # implicit complex record (i.e. some fields are records or unions) @@ -1250,7 +1251,7 @@ def _add_pkg_vars(_vars: Vars) -> Vars: # is the path to the subpackage's # parent context subpkg = Subpkg.from_dfn(dfn) - if subpkg and dfn.name.l != "utl": + if subpkg and dfn.name.l == "utl": vars_["parent_file"] = Var( name="parent_file", _type=Union[str, PathLike], @@ -1535,8 +1536,14 @@ def _metadata() -> List[Metadata]: def _fmt_var(var: Var) -> List[str]: exclude = ["longname", "description"] + + def _fmt_name(k, v): + return v.replace("-", "_") if k == "name" else v + return [ - " ".join([k, v]) for k, v in var.items() if k not in exclude + " ".join([k, _fmt_name(k, v)]).strip() + for k, v in var.items() + if k not in exclude ] meta = dfn.metadata or list() diff --git a/flopy/mf6/utils/templates/attrs.jinja b/flopy/mf6/utils/templates/attrs.jinja index bcc2c280d0..a672be109f 100644 --- a/flopy/mf6/utils/templates/attrs.jinja +++ b/flopy/mf6/utils/templates/attrs.jinja @@ -1,11 +1,11 @@ {%- if base == "MFPackage" %} {% for var in variables.values() if var.class_attr %} {%- if var.kind == "list" or var.kind == "record" or var.kind == "union" %} - {{ var.name }} = ListTemplateGenerator(({% if name.l is not none and name.l != "sim" %}"{{ name.l }}6", {% endif %}"{{ name.r }}", "{{ var.block }}", "{{ var.name }}")) + {{ var.name }} = ListTemplateGenerator(({% if name.l is not none and name.l != "sim" and name.l != "utl" and name.l != "exg" %}"{{ name.l }}6", {% endif %}"{{ name.r }}", "{{ var.block }}", "{{ var.name }}")) {%- elif var.kind == "array" %} - {{ var.name }} = ArrayTemplateGenerator(({% if name.l is not none and name.l != "sim" %}"{{ name.l }}6", {% endif %}"{{ name.r }}", "{{ var.block }}", "{{ var.name }}")) + {{ var.name }} = ArrayTemplateGenerator(({% if name.l is not none and name.l != "sim" and name.l != "utl" and name.l != "exg" %}"{{ name.l }}6", {% endif %}"{{ name.r }}", "{{ var.block }}", "{{ var.name }}")) {%- endif -%} - {%- endfor %} + {%- endfor -%} {% endif -%} {%- if base == "MFModel" %} model_type = "{{ name.title }}" diff --git a/flopy/mf6/utils/templates/context.py.jinja b/flopy/mf6/utils/templates/context.py.jinja index b0402ddc53..16ef6bd37f 100644 --- a/flopy/mf6/utils/templates/context.py.jinja +++ b/flopy/mf6/utils/templates/context.py.jinja @@ -8,6 +8,7 @@ from numpy.typing import NDArray from flopy.mf6.data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenerator from flopy.mf6.mfpackage import MFPackage, MFChildPackages from flopy.mf6.mfmodel import MFModel +{# avoid circular import; some pkgs (e.g. mfnam) are used by mfsimbase.py #} {% if base == "MFSimulationBase" %} from flopy.mf6.mfsimbase import MFSimulationBase {% endif %} @@ -23,64 +24,6 @@ class {% if base == "MFSimulationBase" %}MF{% else %}Modflow{% endif %}{{ name.t {% include "load.jinja" %} -{# inlining all this below since it can ideally be made unnecessary before long? -#} {% if subpkg %} -class {{ name.title.title() }}Packages(MFChildPackages): - """ - {{ name.title.title() }}Packages is a container class for the Modflow{{ name.title.title() }} class. - - Methods - ------- - initialize - Initializes a new Modflow{{ name.title.title() }} package removing any sibling child - packages attached to the same parent package. See Modflow{{ name.title.title() }} init - documentation for definition of parameters. - append_package - Adds a new Modflow{{ name.title.title() }} package to the container. See Modflow{{ name.title.title() }} - init documentation for definition of parameters. - """ - - package_abbr = "{{ name.title.lower() }}packages" - - def initialize( - self, - {%- for name, var in variables.items() if ("simulation" not in name and "model" not in name and "package" not in name and "loading" not in name ) %} - {%- if var._type == "MFSimulation" %} - {{ name }} = None, - {%- elif var.default is defined %} - {{ name }}: {{ var._type }} = {{ var.default }}, - {%- else -%} - {{ name }}: {{ var._type }}, - {% endif -%} - {%- endfor %} - ): - new_package = Modflow{{ name.title.title() }}( - self._cpparent, - {%- for name, var in variables.items() if ("simulation" not in name and "model" not in name and "package" not in name and "loading" not in name ) %} - {{ name }}={{ name }}, - {%- endfor %} - child_builder_call=True, - ) - self.init_package(new_package, filename) - - def append_package( - self, - {%- for name, var in variables.items() if ("simulation" not in name and "model" not in name and "package" not in name and "loading" not in name ) %} - {%- if var._type == "MFSimulation" %} - {{ name }} = None, - {%- elif var.default is defined %} - {{ name }}: {{ var._type }} = {{ var.default }}, - {%- else -%} - {{ name }}: {{ var._type }}, - {% endif -%} - {%- endfor %} - ): - new_package = Modflow{{ name.title.title() }}( - self._cpparent, - {%- for name, var in variables.items() if ("simulation" not in name and "model" not in name and "package" not in name and "loading" not in name) %} - {{ name }}={{ name }}, - {%- endfor %} - child_builder_call=True, - ) - self._append_package(new_package, filename) +{% include "packages.jinja" %} {% endif %} \ No newline at end of file diff --git a/flopy/mf6/utils/templates/docstring_params.jinja b/flopy/mf6/utils/templates/docstring_params.jinja index 184dff93ab..3aeea087c6 100644 --- a/flopy/mf6/utils/templates/docstring_params.jinja +++ b/flopy/mf6/utils/templates/docstring_params.jinja @@ -3,7 +3,7 @@ {%- if v.description is defined and not v.is_choice %} {{ v.description|wordwrap|indent(4 + (loop.depth * 4), first=True) }} {%- endif %} -{%- if v.children is defined and loop.depth < 2 -%} +{%- if v.children is defined and loop.depth < 2-%} {{ loop(v.children.values())|indent(4) }} {%- endif %} {% endfor -%} \ No newline at end of file diff --git a/flopy/mf6/utils/templates/init.jinja b/flopy/mf6/utils/templates/init.jinja index 0833d1a404..804eeb6bce 100644 --- a/flopy/mf6/utils/templates/init.jinja +++ b/flopy/mf6/utils/templates/init.jinja @@ -52,9 +52,6 @@ def __init__( package_type="{{ name.r }}", **kwargs ) - {% if name.l == "exg" -%} - parent.register_exchange_file(self) - {%- endif %} {% for n, var in variables.items() -%} {%- if var.init_assign -%} self.{{ n }} = {{ n }} @@ -71,5 +68,8 @@ def __init__( self.{{ n }} = self.build_mfdata("{% if n == "continue_" %}continue{% else %}{{ n }}{% endif %}", {{ n }}) {% endif -%} {%- endfor -%} + {% if name.l == "exg" -%} + parent.register_exchange_file(self) + {%- endif %} self._init_complete = True {% endif %} \ No newline at end of file diff --git a/flopy/mf6/utils/templates/packages.jinja b/flopy/mf6/utils/templates/packages.jinja new file mode 100644 index 0000000000..9b96b55062 --- /dev/null +++ b/flopy/mf6/utils/templates/packages.jinja @@ -0,0 +1,58 @@ +class {{ name.title.title() }}Packages(MFChildPackages): + """ + {{ name.title.title() }}Packages is a container class for the Modflow{{ name.title.title() }} class. + + Methods + ------- + initialize + Initializes a new Modflow{{ name.title.title() }} package removing any sibling child + packages attached to the same parent package. See Modflow{{ name.title.title() }} init + documentation for definition of parameters. + append_package + Adds a new Modflow{{ name.title.title() }} package to the container. See Modflow{{ name.title.title() }} + init documentation for definition of parameters. + """ + + package_abbr = "{{ name.title.lower() }}packages" + + def initialize( + self, + {%- for n, var in variables.items() if (n != "parent" and n != "loading_package" and var.init_param) %} + {%- if var._type == "MFSimulation" %} + {{ n }} = None, + {%- elif var.default is defined %} + {{ n }}: {{ var._type }} = {{ var.default }}, + {%- else -%} + {{ n }}: {{ var._type }}, + {% endif -%} + {%- endfor %} + ): + new_package = Modflow{{ name.title.title() }}( + self._cpparent, + {%- for n, var in variables.items() if (n != "parent" and n != "loading_package" and var.init_param) %} + {{ n }}={{ n }}, + {%- endfor %} + child_builder_call=True, + ) + self.init_package(new_package, filename) + + def append_package( + self, + {%- for n, var in variables.items() if (n != "parent" and n != "loading_package" and var.init_param) %} + {%- if var._type == "MFSimulation" %} + {{ n }} = None, + {%- elif var.default is defined %} + {{ n }}: {{ var._type }} = {{ var.default }}, + {%- else -%} + {{ n }}: {{ var._type }}, + {% endif -%} + {%- endfor %} + ): + new_package = Modflow{{ name.title.title() }}( + self._cpparent, + {%- for n, var in variables.items() if (n != "parent" and n != "loading_package" and var.init_param) %} + {{ n }}={{ n }}, + {%- endfor %} + child_builder_call=True, + ) + self._append_package(new_package, filename) \ No newline at end of file From abbd94e4ab4bef586c3a0bdca3b34abb8947a4b8 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 3 Oct 2024 08:08:20 -0400 Subject: [PATCH 32/46] about half of test_mf6.py passing --- flopy/mf6/data/mfdatastorage.py | 2 +- flopy/mf6/utils/createpackages.py | 43 +++++++++++++++++++++------ flopy/mf6/utils/templates/attrs.jinja | 8 ++--- flopy/mf6/utils/templates/init.jinja | 8 ++--- 4 files changed, 43 insertions(+), 18 deletions(-) diff --git a/flopy/mf6/data/mfdatastorage.py b/flopy/mf6/data/mfdatastorage.py index 7f66574c55..0d4db95540 100644 --- a/flopy/mf6/data/mfdatastorage.py +++ b/flopy/mf6/data/mfdatastorage.py @@ -316,7 +316,7 @@ def __init__( self.data_structure_type = data_structure_type package_dim = self.data_dimensions.package_dim self.in_model = ( - self.data_dimensions is not None + package_dim is not None and len(package_dim.package_path) > 1 and package_dim.model_dim[0].model_name is not None and package_dim.model_dim[0].model_name.lower() diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index ec0114a44a..f31d6a5692 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -392,7 +392,11 @@ def load_dfn(f, name: Optional[DfnName] = None) -> Dfn: if sep == "flopy": if meta is None: meta = list() - meta.append(tail.strip()) + tail = tail.strip() + if "solution_package" in tail: + tail = tail.split() + tail.pop(1) + meta.append(tail) continue head, sep, tail = line.partition("package-type") if sep == "package-type": @@ -400,6 +404,7 @@ def load_dfn(f, name: Optional[DfnName] = None) -> Dfn: meta = list meta.append(f"{sep} {tail.strip()}") continue + head, sep, tail = line.partition("solution_package") continue # if we hit a newline and the parameter dict @@ -414,6 +419,8 @@ def load_dfn(f, name: Optional[DfnName] = None) -> Dfn: # split the attribute's key and value and # store it in the parameter dictionary key, _, value = line.partition(" ") + if key == "default_value": + key = "default" var[key] = value # add the final parameter @@ -461,10 +468,10 @@ def from_dfn(cls, dfn: Dfn) -> Optional["Subpkg"]: lines = { "subpkg": next( - iter(m for m in dfn.metadata if m.startswith("subpac")), None + iter(m for m in dfn.metadata if isinstance(m, str) and m.startswith("subpac")), None ), "parent": next( - iter(m for m in dfn.metadata if m.startswith("parent")), None + iter(m for m in dfn.metadata if isinstance(m, str) and m.startswith("parent")), None ), } @@ -549,7 +556,7 @@ class Var: init_assign: bool = False init_build: bool = False init_super: bool = False - class_attr: bool = True + class_attr: bool = False def __init__( self, @@ -568,7 +575,7 @@ def __init__( init_assign: bool = False, init_build: bool = False, init_super: bool = False, - class_attr: bool = True, + class_attr: bool = False, ): self.name = name self._type = _type or Any @@ -722,13 +729,13 @@ def _parent() -> Optional[str]: return None if l in ["sim", "exg", "sln"]: return "MFSimulation" - if r == "nam": - return "MFModel" + if r in ["nam"] and name.l is None: + return "MFSimulation" if _subpkg: if len(_subpkg.parents) > 1: return f"Union[{', '.join([_try_get_type_name(t) for t in _subpkg.parents])}]" return _subpkg.parents[0] - return "MFPackage" + return "MFModel" parent = _parent() @@ -771,6 +778,7 @@ def _convert( description = var.get("description", "") children = None is_record = False + class_attr = False def _description(descr: str) -> str: """ @@ -853,6 +861,9 @@ def _fields(record_name: str) -> Vars: # lists which have a consistent record type are # regular, inconsistent record types irregular. if _type.startswith("recarray"): + # flag as a class attribute (ListTemplateGenerator etc) + class_attr = True + # make sure columns are defined names = _type.split()[1:] n_names = len(names) @@ -948,12 +959,18 @@ def _is_implicit_scalar_record(): # union (product), children are record choices elif _type.startswith("keystring"): + # flag as a class attribute (ListTemplateGenerator etc) + class_attr = True + names = _type.split()[1:] children = {n: _convert(dfn[n], wrap=True) for n in names} type_ = Union[tuple([c._type for c in children.values()])] # record (sum) type, children are fields elif _type.startswith("record"): + # flag as a class attribute (ListTemplateGenerator etc) + class_attr = True + children = _fields(_name) if len(children) > 1: record_type = Tuple[ @@ -992,6 +1009,8 @@ def _is_implicit_scalar_record(): # but if it's in a record make it a variadic tuple, # and if its item type is a string use an iterable. elif shape is not None: + # flag as a class attribute (ListTemplateGenerator etc) + class_attr = True scalars = list(_SCALAR_TYPES.keys()) if in_record: if _type not in scalars: @@ -1017,6 +1036,11 @@ def _is_implicit_scalar_record(): # keywords default to False, everything else to None default = var.get("default", False if type_ is bool else None) + if isinstance(default, str) and type_ is not str: + try: + default = eval(default) + except: + pass if _name in ["continue", "print_input"]: # hack... default = None @@ -1034,6 +1058,7 @@ def _is_implicit_scalar_record(): children=children, init_param=True, init_build=True, + class_attr=class_attr ) # check if the variable references a subpackage @@ -1189,8 +1214,8 @@ def _add_exg_vars(_vars: Vars) -> Vars: if not key: continue vars_[subpkg.key].init_param = False + vars_[subpkg.key].init_build = True vars_[subpkg.key].class_attr = True - # vars_[subpkg.key].subpkg = None vars_[subpkg.val] = Var( name=subpkg.val, description=subpkg.description, diff --git a/flopy/mf6/utils/templates/attrs.jinja b/flopy/mf6/utils/templates/attrs.jinja index a672be109f..bca097339a 100644 --- a/flopy/mf6/utils/templates/attrs.jinja +++ b/flopy/mf6/utils/templates/attrs.jinja @@ -1,17 +1,17 @@ {%- if base == "MFPackage" %} {% for var in variables.values() if var.class_attr %} {%- if var.kind == "list" or var.kind == "record" or var.kind == "union" %} - {{ var.name }} = ListTemplateGenerator(({% if name.l is not none and name.l != "sim" and name.l != "utl" and name.l != "exg" %}"{{ name.l }}6", {% endif %}"{{ name.r }}", "{{ var.block }}", "{{ var.name }}")) + {{ var.name }} = ListTemplateGenerator(({% if name.l is not none and name.l != "sim" and name.l != "sln" and name.l != "utl" and name.l != "exg" %}"{{ name.l }}6", {% endif %}"{{ name.r }}", "{{ var.block }}", "{{ var.name }}")) {%- elif var.kind == "array" %} - {{ var.name }} = ArrayTemplateGenerator(({% if name.l is not none and name.l != "sim" and name.l != "utl" and name.l != "exg" %}"{{ name.l }}6", {% endif %}"{{ name.r }}", "{{ var.block }}", "{{ var.name }}")) + {{ var.name }} = ArrayTemplateGenerator(({% if name.l is not none and name.l != "sim" and name.l != "sln" and name.l != "utl" and name.l != "exg" %}"{{ name.l }}6", {% endif %}"{{ name.r }}", "{{ var.block }}", "{{ var.name }}")) {%- endif -%} - {%- endfor -%} + {%- endfor -%} {% endif -%} {%- if base == "MFModel" %} model_type = "{{ name.title }}" {%- elif base == "MFPackage" %} package_abbr = "{% if name.l != "sln" and name.l != "sim" and name.l != "exg" and name.l is not none %}{{ name.l }}{% endif %}{{ name.r }}" _package_type = "{{ name.r }}" - dfn_file_name = "{% if name.l != "sln" and name.l is not none %}{{ name.l }}-{% elif name.l is none %}sim-{% endif %}{{ name.r }}.dfn" + dfn_file_name = "{% if name.l is not none %}{{ name.l }}-{% elif name.l is none %}sim-{% endif %}{{ name.r }}.dfn" dfn = {{ metadata|pprint|indent(10) }} {% endif -%} \ No newline at end of file diff --git a/flopy/mf6/utils/templates/init.jinja b/flopy/mf6/utils/templates/init.jinja index 804eeb6bce..8a19503b4c 100644 --- a/flopy/mf6/utils/templates/init.jinja +++ b/flopy/mf6/utils/templates/init.jinja @@ -62,14 +62,14 @@ def __init__( "{{ var.subpkg.abbr }}", {{ var.subpkg.val }}, "{{ var.subpkg.param }}", - self.{{ var.subpkg.key }} + self._{{ var.subpkg.key }} ) {% elif var.init_build -%} self.{{ n }} = self.build_mfdata("{% if n == "continue_" %}continue{% else %}{{ n }}{% endif %}", {{ n }}) {% endif -%} - {%- endfor -%} - {% if name.l == "exg" -%} + {% if name.l == "exg" and n == "exgmnameb" -%} parent.register_exchange_file(self) - {%- endif %} + {% endif -%} + {%- endfor -%} self._init_complete = True {% endif %} \ No newline at end of file From 56f177c957a0d648b8971dfff7dec081cd51b715 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 3 Oct 2024 10:27:45 -0400 Subject: [PATCH 33/46] extra subpkgs --- autotest/test_createpackages.py | 61 +++++++++++++++++++++++++++++-- flopy/mf6/utils/createpackages.py | 18 +++++++-- 2 files changed, 73 insertions(+), 6 deletions(-) diff --git a/autotest/test_createpackages.py b/autotest/test_createpackages.py index 6d520886e3..6fd231e8f3 100644 --- a/autotest/test_createpackages.py +++ b/autotest/test_createpackages.py @@ -1,3 +1,10 @@ +from ast import AST, expr +from ast import parse as parse_ast +from itertools import zip_longest +from pprint import pformat +from shutil import copy, copytree +from typing import List, Union + import pytest from modflow_devtools.misc import run_cmd @@ -11,7 +18,9 @@ ) PROJ_ROOT = get_project_root_path() -DFN_PATH = PROJ_ROOT / "flopy" / "mf6" / "data" / "dfn" +MF6_PATH = PROJ_ROOT / "flopy" / "mf6" +TGT_PATH = MF6_PATH / "modflow" +DFN_PATH = MF6_PATH / "data" / "dfn" DFN_NAMES = [ dfn.stem for dfn in DFN_PATH.glob("*.dfn") @@ -62,5 +71,51 @@ def test_make_targets(dfn_name, function_tmpdir): def test_make_all(function_tmpdir): make_all(DFN_PATH, function_tmpdir, verbose=True) - run_cmd("ruff", "format", function_tmpdir, verbose=True) - run_cmd("ruff", "check", "--fix", function_tmpdir, verbose=True) + + +def compare_ast( + node1: Union[expr, List[expr]], node2: Union[expr, List[expr]] +) -> bool: + if type(node1) is not type(node2): + return False + + if isinstance(node1, AST): + for k, v in vars(node1).items(): + if k in { + "lineno", + "end_lineno", + "col_offset", + "end_col_offset", + "ctx", + }: + continue + if not compare_ast(v, getattr(node2, k)): + return False + return True + + elif isinstance(node1, list) and isinstance(node2, list): + return all(compare_ast(n1, n2) for n1, n2 in zip_longest(node1, node2)) + else: + return node1 == node2 + + +def test_equivalence(function_tmpdir): + prev_dir = function_tmpdir / "prev" + test_dir = function_tmpdir / "test" + test_dir.mkdir() + copytree(TGT_PATH, prev_dir) + make_all(DFN_PATH, test_dir, verbose=True) + prev_files = list(prev_dir.glob("*.py")) + test_files = list(test_dir.glob("*.py")) + prev_names = set([p.name for p in prev_files]) + test_names = set([p.name for p in test_files]) + diff = prev_names ^ test_names + assert not any(diff), ( + f"previous files don't match test files\n" + f"=> symmetric difference:\n{pformat(diff)}\n" + f"=> prev - test:\n{pformat(prev_names - test_names)}\n" + f"=> test - prev:\n{pformat(test_names - prev_names)}\n" + ) + for prev_file, test_file in zip(prev_files, test_files): + prev = parse_ast(open(prev_file).read()) + test = parse_ast(open(test_file).read()) diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index f31d6a5692..ae2cb4470b 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -336,6 +336,8 @@ def contexts(self) -> List[ContextName]: ContextName(None, self.r), # nam pkg ContextName(*self), # simulation ] + elif self in [("gwf", "mvr"), ("gwf", "gnc"), ("gwt", "mvt")]: + return [ContextName(*self), ContextName(None, self.r)] else: return [ ContextName(*self), # nam pkg @@ -468,10 +470,20 @@ def from_dfn(cls, dfn: Dfn) -> Optional["Subpkg"]: lines = { "subpkg": next( - iter(m for m in dfn.metadata if isinstance(m, str) and m.startswith("subpac")), None + iter( + m + for m in dfn.metadata + if isinstance(m, str) and m.startswith("subpac") + ), + None, ), "parent": next( - iter(m for m in dfn.metadata if isinstance(m, str) and m.startswith("parent")), None + iter( + m + for m in dfn.metadata + if isinstance(m, str) and m.startswith("parent") + ), + None, ), } @@ -1058,7 +1070,7 @@ def _is_implicit_scalar_record(): children=children, init_param=True, init_build=True, - class_attr=class_attr + class_attr=class_attr, ) # check if the variable references a subpackage From 4b285d5bbe262df0242902607afb6c3b8e11ffac Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 3 Oct 2024 11:49:51 -0400 Subject: [PATCH 34/46] ast equiv test, some mf6 tests still failing (adv pkgs?) --- autotest/test_createpackages.py | 61 +++++++++++++++------- flopy/mf6/data/dfn/exg-gwfgwf.dfn | 1 + flopy/mf6/data/dfn/gwf-nam.dfn | 1 + flopy/mf6/data/dfn/gwf-npf.dfn | 2 + flopy/mf6/data/dfn/gwt-uzt.dfn | 5 +- flopy/mf6/data/dfn/utl-obs.dfn | 1 + flopy/mf6/data/dfn/utl-ts.dfn | 8 +++ flopy/mf6/utils/createpackages.py | 24 +++++---- flopy/mf6/utils/templates/context.py.jinja | 2 +- flopy/mf6/utils/templates/packages.jinja | 4 +- 10 files changed, 76 insertions(+), 33 deletions(-) diff --git a/autotest/test_createpackages.py b/autotest/test_createpackages.py index 6fd231e8f3..41183b0035 100644 --- a/autotest/test_createpackages.py +++ b/autotest/test_createpackages.py @@ -1,9 +1,10 @@ -from ast import AST, expr +from ast import AST, Assign, ClassDef, expr from ast import parse as parse_ast from itertools import zip_longest from pprint import pformat from shutil import copy, copytree from typing import List, Union +from warnings import warn import pytest from modflow_devtools.misc import run_cmd @@ -76,27 +77,34 @@ def test_make_all(function_tmpdir): def compare_ast( node1: Union[expr, List[expr]], node2: Union[expr, List[expr]] ) -> bool: - if type(node1) is not type(node2): + t1 = type(node1) + t2 = type(node2) + if t1 is not t2: + print(f"[type mismatch] {t1} != {t2}") return False - if isinstance(node1, AST): - for k, v in vars(node1).items(): - if k in { - "lineno", - "end_lineno", - "col_offset", - "end_col_offset", - "ctx", - }: - continue - if not compare_ast(v, getattr(node2, k)): - return False - return True - - elif isinstance(node1, list) and isinstance(node2, list): - return all(compare_ast(n1, n2) for n1, n2 in zip_longest(node1, node2)) - else: - return node1 == node2 + if t1 is ClassDef: + assert t2 is ClassDef + assert node1.name == node2.name + for base1, base2 in zip(node1.bases, node2.bases): + assert base1.attr == base2.id # hack.. + + body1, body2 = node1.body, node2.body + assert len(body1) == len(body2), f"bodies don't match in {node1.name}" + + for b1, b2 in zip(body1, body2): + if isinstance(b1, Assign): + assert isinstance(b2, Assign) + b1tgts = set(sorted([t.id for t in b1.targets])) + b2tgts = set(sorted([t.id for t in b2.targets])) + diff = b1tgts ^ b2tgts + if any(diff): + warn( + f"targets don't match for assignment in {node1.name}\n" + f"=> symmetric difference:\n{pformat(diff)}\n" + f"=> prev - test:\n{pformat(b1tgts - b2tgts)}\n" + f"=> test - prev:\n{pformat(b2tgts - b1tgts)}\n" + ) def test_equivalence(function_tmpdir): @@ -119,3 +127,16 @@ def test_equivalence(function_tmpdir): for prev_file, test_file in zip(prev_files, test_files): prev = parse_ast(open(prev_file).read()) test = parse_ast(open(test_file).read()) + prev_classes = [n for n in prev.body if isinstance(n, ClassDef)] + test_classes = [n for n in test.body if isinstance(n, ClassDef)] + prev_clsnames = set([c.name for c in prev_classes]) + test_clsnames = set([c.name for c in test_classes]) + diff = prev_clsnames ^ test_clsnames + assert not any(diff), ( + f"previous classes don't match test classes in {test_file.name}\n" + f"=> symmetric difference:\n{pformat(diff)}\n" + f"=> prev - test:\n{pformat(prev_clsnames - test_clsnames)}\n" + f"=> test - prev:\n{pformat(test_clsnames - prev_clsnames)}\n" + ) + for prev_cls, test_cls in zip(prev_classes, test_classes): + compare_ast(prev_cls, test_cls) diff --git a/flopy/mf6/data/dfn/exg-gwfgwf.dfn b/flopy/mf6/data/dfn/exg-gwfgwf.dfn index 0f68acead6..02caa91b21 100644 --- a/flopy/mf6/data/dfn/exg-gwfgwf.dfn +++ b/flopy/mf6/data/dfn/exg-gwfgwf.dfn @@ -61,6 +61,7 @@ name cvoptions type record variablecv dewatered reader urword optional true +class_attr false longname vertical conductance options description none diff --git a/flopy/mf6/data/dfn/gwf-nam.dfn b/flopy/mf6/data/dfn/gwf-nam.dfn index f4e6ba4839..9ef47195f3 100644 --- a/flopy/mf6/data/dfn/gwf-nam.dfn +++ b/flopy/mf6/data/dfn/gwf-nam.dfn @@ -38,6 +38,7 @@ name newtonoptions type record newton under_relaxation reader urword optional true +class_attr false longname newton keyword and options description none diff --git a/flopy/mf6/data/dfn/gwf-npf.dfn b/flopy/mf6/data/dfn/gwf-npf.dfn index 23cb314c2a..3a2a7284a3 100644 --- a/flopy/mf6/data/dfn/gwf-npf.dfn +++ b/flopy/mf6/data/dfn/gwf-npf.dfn @@ -42,6 +42,7 @@ name cvoptions type record variablecv dewatered reader urword optional true +class_attr false longname vertical conductance options description none @@ -123,6 +124,7 @@ name xt3doptions type record xt3d rhs reader urword optional true +class_attr false longname keyword to activate XT3D description none diff --git a/flopy/mf6/data/dfn/gwt-uzt.dfn b/flopy/mf6/data/dfn/gwt-uzt.dfn index 00524848bd..9898b610d5 100644 --- a/flopy/mf6/data/dfn/gwt-uzt.dfn +++ b/flopy/mf6/data/dfn/gwt-uzt.dfn @@ -16,6 +16,7 @@ type string shape (naux) reader urword optional true +class_attr true longname keyword to specify aux variables description REPLACE auxnames {'{#1}': 'Groundwater Transport'} @@ -399,7 +400,7 @@ description real or character value that defines the concentration of unsaturate block period name auxiliaryrecord -type record auxiliary auxname auxval +type record aux auxname auxval shape tagged in_record true @@ -408,7 +409,7 @@ longname description block period -name auxiliary +name aux type keyword shape in_record true diff --git a/flopy/mf6/data/dfn/utl-obs.dfn b/flopy/mf6/data/dfn/utl-obs.dfn index d75ce62e47..0d0cb9ddd6 100644 --- a/flopy/mf6/data/dfn/utl-obs.dfn +++ b/flopy/mf6/data/dfn/utl-obs.dfn @@ -29,6 +29,7 @@ shape block_variable true in_record false reader urword +class_attr false optional false longname description diff --git a/flopy/mf6/data/dfn/utl-ts.dfn b/flopy/mf6/data/dfn/utl-ts.dfn index a7165ea382..cb641256f2 100644 --- a/flopy/mf6/data/dfn/utl-ts.dfn +++ b/flopy/mf6/data/dfn/utl-ts.dfn @@ -20,6 +20,7 @@ type keyword shape reader urword optional false +in_record true longname description xxx @@ -30,6 +31,7 @@ shape any1d tagged false reader urword optional false +in_record true longname description Name by which a package references a particular time-array series. The name must be unique among all time-array series used in a package. @@ -49,6 +51,7 @@ type keyword shape reader urword optional false +in_record true longname description xxx @@ -59,6 +62,7 @@ valid stepwise linear linearend shape time_series_names tagged false reader urword +in_record true optional false longname description Interpolation method, which is either STEPWISE or LINEAR. @@ -108,6 +112,7 @@ name sfacs type keyword shape reader urword +in_record true optional false longname description xxx @@ -119,6 +124,7 @@ shape List[ContextName]: ContextName(None, self.r), # nam pkg ContextName(*self), # simulation ] - elif self in [("gwf", "mvr"), ("gwf", "gnc"), ("gwt", "mvt")]: - return [ContextName(*self), ContextName(None, self.r)] else: return [ ContextName(*self), # nam pkg ContextName(self.l, None), # model ] + elif (self.l, self.r) in [ + ("gwf", "mvr"), + ("gwf", "gnc"), + ("gwt", "mvt"), + ]: + return [ContextName(*self), ContextName(None, self.r)] return [ContextName(*self)] @@ -423,6 +427,8 @@ def load_dfn(f, name: Optional[DfnName] = None) -> Dfn: key, _, value = line.partition(" ") if key == "default_value": key = "default" + if value in ["true", "false"]: + value = value == "true" var[key] = value # add the final parameter @@ -786,11 +792,11 @@ def _convert( shape = None if shape == "" else shape optional = var.get("optional", True) in_record = var.get("in_record", False) - tagged = var.get("tagged, False") + tagged = var.get("tagged", False) description = var.get("description", "") children = None is_record = False - class_attr = False + class_attr = var.get("class_attr", False) def _description(descr: str) -> str: """ @@ -874,7 +880,7 @@ def _fields(record_name: str) -> Vars: # regular, inconsistent record types irregular. if _type.startswith("recarray"): # flag as a class attribute (ListTemplateGenerator etc) - class_attr = True + class_attr = var.get("class_attr", True) # make sure columns are defined names = _type.split()[1:] @@ -972,7 +978,7 @@ def _is_implicit_scalar_record(): # union (product), children are record choices elif _type.startswith("keystring"): # flag as a class attribute (ListTemplateGenerator etc) - class_attr = True + class_attr = var.get("class_attr", True) names = _type.split()[1:] children = {n: _convert(dfn[n], wrap=True) for n in names} @@ -981,7 +987,7 @@ def _is_implicit_scalar_record(): # record (sum) type, children are fields elif _type.startswith("record"): # flag as a class attribute (ListTemplateGenerator etc) - class_attr = True + class_attr = var.get("class_attr", True) children = _fields(_name) if len(children) > 1: @@ -1022,7 +1028,7 @@ def _is_implicit_scalar_record(): # and if its item type is a string use an iterable. elif shape is not None: # flag as a class attribute (ListTemplateGenerator etc) - class_attr = True + class_attr = var.get("class_attr", True) scalars = list(_SCALAR_TYPES.keys()) if in_record: if _type not in scalars: @@ -1578,7 +1584,7 @@ def _fmt_name(k, v): return v.replace("-", "_") if k == "name" else v return [ - " ".join([k, _fmt_name(k, v)]).strip() + " ".join([k, str(_fmt_name(k, v))]).strip() for k, v in var.items() if k not in exclude ] diff --git a/flopy/mf6/utils/templates/context.py.jinja b/flopy/mf6/utils/templates/context.py.jinja index 16ef6bd37f..b8718f7292 100644 --- a/flopy/mf6/utils/templates/context.py.jinja +++ b/flopy/mf6/utils/templates/context.py.jinja @@ -24,6 +24,6 @@ class {% if base == "MFSimulationBase" %}MF{% else %}Modflow{% endif %}{{ name.t {% include "load.jinja" %} -{% if subpkg %} +{% if subpkg and name.r != "hpc" %} {% include "packages.jinja" %} {% endif %} \ No newline at end of file diff --git a/flopy/mf6/utils/templates/packages.jinja b/flopy/mf6/utils/templates/packages.jinja index 9b96b55062..b1dfbaf72f 100644 --- a/flopy/mf6/utils/templates/packages.jinja +++ b/flopy/mf6/utils/templates/packages.jinja @@ -36,6 +36,7 @@ class {{ name.title.title() }}Packages(MFChildPackages): ) self.init_package(new_package, filename) +{% if name.r != "obs" %} def append_package( self, {%- for n, var in variables.items() if (n != "parent" and n != "loading_package" and var.init_param) %} @@ -55,4 +56,5 @@ class {{ name.title.title() }}Packages(MFChildPackages): {%- endfor %} child_builder_call=True, ) - self._append_package(new_package, filename) \ No newline at end of file + self._append_package(new_package, filename) +{% endif %} \ No newline at end of file From 656ff894d83476ac82b29641f01f1f1b5157300f Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 3 Oct 2024 13:23:48 -0400 Subject: [PATCH 35/46] no py38 --- .github/workflows/commit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml index 07e7b73bcc..b28e3af4df 100644 --- a/.github/workflows/commit.yml +++ b/.github/workflows/commit.yml @@ -133,7 +133,7 @@ jobs: fail-fast: false matrix: os: [ ubuntu-latest, macos-latest, windows-latest ] - python-version: [ 3.8, 3.9, "3.10", "3.11", "3.12" ] + python-version: [ 3.9, "3.10", "3.11", "3.12" ] defaults: run: shell: bash -l {0} From f850921d23cb65fbfea66c250cd92bf433bf6fbb Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 3 Oct 2024 13:40:35 -0400 Subject: [PATCH 36/46] appease codacy: use literal_eval, remove unused vars --- flopy/mf6/utils/createpackages.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index 00c1634fc0..dcc6e5e8b9 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -83,6 +83,7 @@ import collections import os +from ast import literal_eval from collections import UserDict, namedtuple from dataclasses import asdict, dataclass, replace from enum import Enum @@ -274,7 +275,7 @@ def title(self) -> str: @property def base(self) -> str: """Base class from which the input context should inherit.""" - l, r = self + _, r = self if self == ("sim", "nam"): return "MFSimulationBase" if r is None: @@ -378,9 +379,7 @@ def __init__( def load_dfn(f, name: Optional[DfnName] = None) -> Dfn: """ - Load an input definition file. Returns a tuple containing - a dictionary of variable specifications as well as a list - of metadata attributes. + Load an input definition from a definition file. """ meta = None @@ -404,7 +403,7 @@ def load_dfn(f, name: Optional[DfnName] = None) -> Dfn: tail.pop(1) meta.append(tail) continue - head, sep, tail = line.partition("package-type") + _, sep, tail = line.partition("package-type") if sep == "package-type": if meta is None: meta = list @@ -808,7 +807,7 @@ def _description(descr: str) -> str: _, replace, tail = descr.strip().partition("REPLACE") if replace: key, _, replacements = tail.strip().partition(" ") - replacements = eval(replacements) + replacements = literal_eval(replacements) common_var = common.get(key, None) if common_var is None: raise ValueError(f"Common variable not found: {key}") @@ -1056,7 +1055,7 @@ def _is_implicit_scalar_record(): default = var.get("default", False if type_ is bool else None) if isinstance(default, str) and type_ is not str: try: - default = eval(default) + default = literal_eval(default) except: pass if _name in ["continue", "print_input"]: # hack... @@ -1616,7 +1615,7 @@ def make_contexts( _TEMPLATE_ENV = Environment( - loader=PackageLoader("flopy", "mf6/utils/templates/") + loader=PackageLoader("flopy", "mf6/utils/templates/"), ) From c34e1469fb508e4aa03499c84caaa6476d24a328 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 3 Oct 2024 13:45:18 -0400 Subject: [PATCH 37/46] unused var --- flopy/mf6/utils/createpackages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index dcc6e5e8b9..995246088b 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -409,7 +409,7 @@ def load_dfn(f, name: Optional[DfnName] = None) -> Dfn: meta = list meta.append(f"{sep} {tail.strip()}") continue - head, sep, tail = line.partition("solution_package") + _, sep, tail = line.partition("solution_package") continue # if we hit a newline and the parameter dict From 5b2e25779b64bfeb591eb605e6ff9aed084fc972 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 3 Oct 2024 14:12:26 -0400 Subject: [PATCH 38/46] multidict to the rescue for duplicate var names --- autotest/test_createpackages.py | 8 ++++---- flopy/mf6/utils/createpackages.py | 23 ++++++++++++++--------- pyproject.toml | 1 + 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/autotest/test_createpackages.py b/autotest/test_createpackages.py index 41183b0035..8f57e88b67 100644 --- a/autotest/test_createpackages.py +++ b/autotest/test_createpackages.py @@ -80,17 +80,17 @@ def compare_ast( t1 = type(node1) t2 = type(node2) if t1 is not t2: - print(f"[type mismatch] {t1} != {t2}") + print(f"type mismatch: {t1} != {t2}") return False if t1 is ClassDef: assert t2 is ClassDef assert node1.name == node2.name for base1, base2 in zip(node1.bases, node2.bases): - assert base1.attr == base2.id # hack.. + assert base1.id == base2.id body1, body2 = node1.body, node2.body - assert len(body1) == len(body2), f"bodies don't match in {node1.name}" + assert len(body1) == len(body2), f"body mismatch in {node1.name}" for b1, b2 in zip(body1, body2): if isinstance(b1, Assign): @@ -100,7 +100,7 @@ def compare_ast( diff = b1tgts ^ b2tgts if any(diff): warn( - f"targets don't match for assignment in {node1.name}\n" + f"assignment targets don't match in {node1.name}\n" f"=> symmetric difference:\n{pformat(diff)}\n" f"=> prev - test:\n{pformat(b1tgts - b2tgts)}\n" f"=> test - prev:\n{pformat(b2tgts - b1tgts)}\n" diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index 995246088b..cad9fae963 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -108,6 +108,7 @@ from warnings import warn import numpy as np +from boltons.dictutils import OMD from jinja2 import Environment, PackageLoader from modflow_devtools.misc import run_cmd from numpy.typing import ArrayLike, NDArray @@ -365,13 +366,14 @@ class Dfn(UserDict): def __init__( self, - variables: Dict[str, Dict[str, str]], + variables: Iterable[Tuple[str, Dict[str, Any]]], name: Optional[DfnName] = None, metadata: Optional[Metadata] = None, ): - super().__init__(variables) + self.omd = OMD(variables) self.name = name self.metadata = metadata + super().__init__(self.omd) Dfns = Dict[str, Dfn] @@ -383,7 +385,7 @@ def load_dfn(f, name: Optional[DfnName] = None) -> Dfn: """ meta = None - vars_ = dict() + vars_ = list() var = dict() for line in f: @@ -417,7 +419,8 @@ def load_dfn(f, name: Optional[DfnName] = None) -> Dfn: # block of attributes if not any(line): if any(var): - vars_[var["name"]] = var + n = var["name"] + vars_.append((n, var)) var = dict() continue @@ -432,7 +435,8 @@ def load_dfn(f, name: Optional[DfnName] = None) -> Dfn: # add the final parameter if any(var): - vars_[var["name"]] = var + n = var["name"] + vars_.append((n, var)) return Dfn(variables=vars_, name=name, metadata=meta) @@ -757,7 +761,7 @@ def _parent() -> Optional[str]: parent = _parent() def _convert( - var: Dict[str, str], + var: Dict[str, Any], wrap: bool = False, ) -> Var: """ @@ -796,6 +800,7 @@ def _convert( children = None is_record = False class_attr = var.get("class_attr", False) + init_build = var.get("init_build", True) def _description(descr: str) -> str: """ @@ -1074,7 +1079,7 @@ def _is_implicit_scalar_record(): default=default, children=children, init_param=True, - init_build=True, + init_build=init_build, class_attr=class_attr, ) @@ -1576,7 +1581,7 @@ def _metadata() -> List[Metadata]: Python, consolidating nested types, etc. """ - def _fmt_var(var: Var) -> List[str]: + def _fmt_var(var: Union[Var, List[Var]]) -> List[str]: exclude = ["longname", "description"] def _fmt_name(k, v): @@ -1590,7 +1595,7 @@ def _fmt_name(k, v): meta = dfn.metadata or list() return [["header"] + [m for m in meta]] + [ - _fmt_var(var) for var in dfn.values() + _fmt_var(var) for var in dfn.omd.values(multi=True) ] return Context( diff --git a/pyproject.toml b/pyproject.toml index 436cbfcf27..5ed2c83362 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ classifiers = [ ] requires-python = ">=3.8" dependencies = [ + "boltons", "Jinja2>=3.0", "numpy>=1.20.3", "matplotlib >=1.4.0", From ede48dd0355c7aa2a0a5a34d92fbaa690f62995b Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 3 Oct 2024 14:40:58 -0400 Subject: [PATCH 39/46] fix test --- autotest/test_createpackages.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/autotest/test_createpackages.py b/autotest/test_createpackages.py index 8f57e88b67..c5c52949b3 100644 --- a/autotest/test_createpackages.py +++ b/autotest/test_createpackages.py @@ -87,7 +87,15 @@ def compare_ast( assert t2 is ClassDef assert node1.name == node2.name for base1, base2 in zip(node1.bases, node2.bases): - assert base1.id == base2.id + def _id(b): + attrs = ["id", "name", "attr"] + for attr in attrs: + try: + return getattr(b, attr) + except: + pass + return None + assert _id(base1) == _id(base2) body1, body2 = node1.body, node2.body assert len(body1) == len(body2), f"body mismatch in {node1.name}" From cd9cbcced8cf1d3f18d35b60e09c2c7640ede337 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 3 Oct 2024 14:55:08 -0400 Subject: [PATCH 40/46] dfn fixes --- flopy/mf6/data/dfn/utl-obs.dfn | 2 ++ flopy/mf6/data/dfn/utl-ts.dfn | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/flopy/mf6/data/dfn/utl-obs.dfn b/flopy/mf6/data/dfn/utl-obs.dfn index 0d0cb9ddd6..474c8b22b3 100644 --- a/flopy/mf6/data/dfn/utl-obs.dfn +++ b/flopy/mf6/data/dfn/utl-obs.dfn @@ -30,6 +30,8 @@ block_variable true in_record false reader urword class_attr false +init_build false +init_param false optional false longname description diff --git a/flopy/mf6/data/dfn/utl-ts.dfn b/flopy/mf6/data/dfn/utl-ts.dfn index cb641256f2..86ea2dceb5 100644 --- a/flopy/mf6/data/dfn/utl-ts.dfn +++ b/flopy/mf6/data/dfn/utl-ts.dfn @@ -83,6 +83,8 @@ type keyword shape reader urword optional false +init_build false +init_param false longname description xxx @@ -93,6 +95,8 @@ valid stepwise linear linearend shape tagged false reader urword +init_build false +init_param false optional false longname description Interpolation method, which is either STEPWISE or LINEAR. @@ -145,6 +149,8 @@ shape tagged false reader urword optional false +init_build false +init_param false longname description xxx From 8e445fd921ec8ef4868f82a7b5b44b2c665cb6f8 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 3 Oct 2024 15:07:33 -0400 Subject: [PATCH 41/46] fixes for mf6 tests.. 127 passing, 20 to go --- autotest/test_createpackages.py | 2 ++ flopy/mf6/utils/createpackages.py | 2 +- flopy/mf6/utils/templates/init.jinja | 25 +++++++++---------------- 3 files changed, 12 insertions(+), 17 deletions(-) diff --git a/autotest/test_createpackages.py b/autotest/test_createpackages.py index c5c52949b3..a23ed1aebc 100644 --- a/autotest/test_createpackages.py +++ b/autotest/test_createpackages.py @@ -87,6 +87,7 @@ def compare_ast( assert t2 is ClassDef assert node1.name == node2.name for base1, base2 in zip(node1.bases, node2.bases): + def _id(b): attrs = ["id", "name", "attr"] for attr in attrs: @@ -95,6 +96,7 @@ def _id(b): except: pass return None + assert _id(base1) == _id(base2) body1, body2 = node1.body, node2.body diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index cad9fae963..86dbe1c684 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -750,7 +750,7 @@ def _parent() -> Optional[str]: return None if l in ["sim", "exg", "sln"]: return "MFSimulation" - if r in ["nam"] and name.l is None: + if name.r is None: return "MFSimulation" if _subpkg: if len(_subpkg.parents) > 1: diff --git a/flopy/mf6/utils/templates/init.jinja b/flopy/mf6/utils/templates/init.jinja index 8a19503b4c..7aca7f7082 100644 --- a/flopy/mf6/utils/templates/init.jinja +++ b/flopy/mf6/utils/templates/init.jinja @@ -1,5 +1,8 @@ def __init__( self, + {% if parent and base != "MFModel" and name.l != "exg" -%} + {{ parent|lower|replace("mf", "")|replace("union", "")|replace("[", "")|replace("]", "")|replace(", ", "_or_") }} = None, + {%- endif %} {%- for name, var in variables.items() if var.init_param %} {%- if var._type == "MFSimulation" %} {{ name }} = None, @@ -11,7 +14,11 @@ def __init__( {%- endfor %} **kwargs, ): -{% if base == "MFSimulationBase" %} + {% if parent and base != "MFModel" and name.l != "exg" -%} + if {{ parent|lower|replace("mf", "")|replace("union", "")|replace("[", "")|replace("]", "")|replace(", ", "_or_") }}: + parent = {{ parent|lower|replace("mf", "")|replace("union", "")|replace("[", "")|replace("]", "")|replace(", ", "_or_") }} + {%- endif %} +{% if base == "MFSimulationBase" or base == "MFModel" %} super().__init__( {%- for n, var in variables.items() if var.init_super %} {{ n }}={{ n }}, @@ -23,27 +30,13 @@ def __init__( self.name_file.{{ n }}.set_data({{ n }}) self.{{ n }} = self.name_file.{{ n }} {% endif -%} - {%- if var.subpkg is defined %} + {%- if var.subpkg is defined and base != "MFModel" %} self.{{ var.subpkg.param }} = self._create_package( "{{ var.subpkg.abbr }}", {{ var.subpkg.param }} ) {% endif -%} {% endfor -%} -{% elif base == "MFModel" %} - super().__init__( - {%- for n, var in variables.items() if var.init_super %} - {{ n }}={{ n }}, - {%- endfor %} - model_type="{{ name.l }}6", - **kwargs, - ) - {%- for n, var in variables.items() %} - {%- if var.block == "options" and var.init_build %} - self.name_file.{{ n }}.set_data({{ n }}) - self.{{ n }} = self.name_file.{{ n }} - {% endif -%} - {%- endfor %} {% elif base == "MFPackage" %} super().__init__( {%- for n, var in variables.items() if var.init_super %} From 951e930888bab632731f485732c2341760c40c95 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 3 Oct 2024 22:15:53 -0400 Subject: [PATCH 42/46] ci --- .github/workflows/commit.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml index b28e3af4df..6a2d8292dc 100644 --- a/.github/workflows/commit.yml +++ b/.github/workflows/commit.yml @@ -181,10 +181,13 @@ jobs: working-directory: autotest run: | pytest -v -m="not example" -n=auto --cov=flopy --cov-append --cov-report=xml --durations=0 --keep-failed=.failed --dist loadfile - coverage report env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Report coverage + working-directory: autotest + run: coverage report + - name: Upload failed test outputs uses: actions/upload-artifact@v4 if: failure() From 3d8c25752ca46e279d1d30d94f1a7eb86ea72c70 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 3 Oct 2024 22:40:11 -0400 Subject: [PATCH 43/46] render-time transformations --- flopy/mf6/utils/createpackages.py | 53 +++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 17 deletions(-) diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index 86dbe1c684..dcf9fca42c 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -92,6 +92,7 @@ from pathlib import Path from typing import ( Any, + Callable, Dict, ForwardRef, Iterable, @@ -155,26 +156,36 @@ def renderable( *, wrap_str: Optional[List[str]] = None, keep_none: Optional[List[str]] = None, + transform: Optional[Dict[str, Callable[[Any], Any]]] = None, ): """ An object meant to be passed into a template as a "rendered" dictionary, where "rendering" means transforming key/value pairs to a form - more convenient for use within the template. - - The object *must* be a dataclass. + appropriate for use within the template. Notes ----- - Jinja supports attribute- and dictionary- + Transformations might be for convenience* or + to handle special cases where a variable has + edge cases or other need for alteration**. + + *Jinja supports attribute- and dictionary- based access but no arbitrary expressions, and only a limited set of custom filters. This can make it awkward to express some - things, so convert the dataclasses we'll - pass to `template.render(...)` to dicts, - with a few touchups. + things. - These include: + **This is convenient for handling complexity + incidental to the current mf6 data framework. + Transforming values at render time helps to + isolate special cases from the more general + templating infrastructure, so the framework + can be refactored more easily over time. + + The object *must* be a dataclass. + + Common use cases include: - converting types to suitably qualified type names - optionally removing key/value pairs whose value is None - optionally quoting strings forming the RHS of an assignment or @@ -190,22 +201,26 @@ def _render(d: dict) -> dict: def _render_key(k): return k - def _render_val(v): - return _try_get_type_name(_try_get_enum_value(v)) + def _render_val(k, v): + v = _try_get_type_name(_try_get_enum_value(v)) + + def noop(v): + return v + + return (transform.get(k, noop) if transform else noop)(v) # drop nones except where keep requested _d = { - _render_key(k): _render_val(v) + _render_key(k): _render_val(k, v) for k, v in d.items() if (k in keep_none or v is not None) } # wrap string values where requested - if wrap_str: - for k in wrap_str: - v = _d.get(k, None) - if v is not None and isinstance(v, str): - _d[k] = f'"{v}"' + for k in wrap_str: + v = _d.get(k, None) + if v is not None and isinstance(v, str): + _d[k] = f'"{v}"' return _d @@ -641,7 +656,11 @@ def __init__( Vars = Dict[str, Var] -@renderable(wrap_str=["default"], keep_none=["block", "default"]) +@renderable( + wrap_str=["default"], + keep_none=["block", "default"], + # TODO replace the flags on Var with transforms? +) @dataclass class Context: """ From 5e46278fa775bbd59d9c62ed660f3be36d966a4b Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Fri, 4 Oct 2024 16:39:49 -0400 Subject: [PATCH 44/46] cleanup --- autotest/test_createpackages.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/autotest/test_createpackages.py b/autotest/test_createpackages.py index a23ed1aebc..81277cf10b 100644 --- a/autotest/test_createpackages.py +++ b/autotest/test_createpackages.py @@ -1,8 +1,7 @@ -from ast import AST, Assign, ClassDef, expr +from ast import Assign, ClassDef, expr from ast import parse as parse_ast -from itertools import zip_longest from pprint import pformat -from shutil import copy, copytree +from shutil import copytree from typing import List, Union from warnings import warn From befdfc7736c98a0f44be916a544844fcfed1b139 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Mon, 7 Oct 2024 15:19:51 -0400 Subject: [PATCH 45/46] module reorg --- ...test_createpackages.py => test_codegen.py} | 32 +- flopy/mf6/utils/codegen/context.py | 666 +++++++ flopy/mf6/utils/codegen/dfn.py | 119 ++ flopy/mf6/utils/codegen/make.py | 98 + flopy/mf6/utils/codegen/ref.py | 88 + flopy/mf6/utils/codegen/render.py | 169 ++ flopy/mf6/utils/codegen/shim.py | 472 +++++ flopy/mf6/utils/codegen/spec.py | 88 + flopy/mf6/utils/codegen/utils.py | 41 + flopy/mf6/utils/createpackages.py | 1650 +---------------- .../utils/templates/docstring_params.jinja | 4 +- flopy/mf6/utils/templates/init.jinja | 22 +- flopy/mf6/utils/templates/records.jinja | 2 +- pyproject.toml | 4 +- 14 files changed, 1784 insertions(+), 1671 deletions(-) rename autotest/{test_createpackages.py => test_codegen.py} (85%) create mode 100644 flopy/mf6/utils/codegen/context.py create mode 100644 flopy/mf6/utils/codegen/dfn.py create mode 100644 flopy/mf6/utils/codegen/make.py create mode 100644 flopy/mf6/utils/codegen/ref.py create mode 100644 flopy/mf6/utils/codegen/render.py create mode 100644 flopy/mf6/utils/codegen/shim.py create mode 100644 flopy/mf6/utils/codegen/spec.py create mode 100644 flopy/mf6/utils/codegen/utils.py diff --git a/autotest/test_createpackages.py b/autotest/test_codegen.py similarity index 85% rename from autotest/test_createpackages.py rename to autotest/test_codegen.py index 81277cf10b..1e76c5d393 100644 --- a/autotest/test_createpackages.py +++ b/autotest/test_codegen.py @@ -2,6 +2,7 @@ from ast import parse as parse_ast from pprint import pformat from shutil import copytree +import traceback from typing import List, Union from warnings import warn @@ -9,11 +10,13 @@ from modflow_devtools.misc import run_cmd from autotest.conftest import get_project_root_path -from flopy.mf6.utils.createpackages import ( +from flopy.mf6.utils.codegen.context import get_context_names +from flopy.mf6.utils.codegen.make import ( DfnName, load_dfn, make_all, make_context, + make_contexts, make_targets, ) @@ -36,7 +39,7 @@ def test_load_dfn(dfn_name): @pytest.mark.parametrize( - "dfn_name, n_flat, n_params", [("gwf-ic", 2, 6), ("prt-prp", 40, 22)] + "dfn_name, n_flat, n_params", [("gwf-ic", 2, 2), ("prt-prp", 40, 18)] ) def test_make_context(dfn_name, n_flat, n_params): with open(DFN_PATH / "common.dfn") as f: @@ -46,13 +49,23 @@ def test_make_context(dfn_name, n_flat, n_params): dfn_name = DfnName(*dfn_name.split("-")) dfn = load_dfn(f, name=dfn_name) - ctx_name = dfn_name.contexts[0] - context = make_context(ctx_name, dfn, common=common) - assert len(dfn_name.contexts) == 1 + context_names = get_context_names(dfn_name) + context_name = context_names[0] + context = make_context(context_name, dfn, common=common) + assert len(context_names) == 1 assert len(context.variables) == n_params assert len(context.metadata) == n_flat + 1 # +1 for metadata +@pytest.mark.skip(reason="TODO") +@pytest.mark.parametrize("dfn_name", ["gwf-ic", "prt-prp", "gwf-nam"]) +def test_make_contexts(dfn_name): + with open(DFN_PATH / "common.dfn") as f: + common = load_dfn(f) + + # TODO + + @pytest.mark.parametrize("dfn_name", DFN_NAMES) def test_make_targets(dfn_name, function_tmpdir): with open(DFN_PATH / "common.dfn") as f: @@ -63,7 +76,7 @@ def test_make_targets(dfn_name, function_tmpdir): dfn = load_dfn(f, name=dfn_name) make_targets(dfn, function_tmpdir, common=common) - for ctx_name in dfn_name.contexts: + for ctx_name in get_context_names(dfn_name): run_cmd("ruff", "format", function_tmpdir, verbose=True) run_cmd("ruff", "check", "--fix", function_tmpdir, verbose=True) assert (function_tmpdir / ctx_name.target).is_file() @@ -135,7 +148,12 @@ def test_equivalence(function_tmpdir): ) for prev_file, test_file in zip(prev_files, test_files): prev = parse_ast(open(prev_file).read()) - test = parse_ast(open(test_file).read()) + try: + test = parse_ast(open(test_file).read()) + except: + raise ValueError( + f"Failed to parse {test_file}: {traceback.format_exc()}" + ) prev_classes = [n for n in prev.body if isinstance(n, ClassDef)] test_classes = [n for n in test.body if isinstance(n, ClassDef)] prev_clsnames = set([c.name for c in prev_classes]) diff --git a/flopy/mf6/utils/codegen/context.py b/flopy/mf6/utils/codegen/context.py new file mode 100644 index 0000000000..77041d68ea --- /dev/null +++ b/flopy/mf6/utils/codegen/context.py @@ -0,0 +1,666 @@ +from ast import literal_eval +from collections import namedtuple +from dataclasses import dataclass, replace +from keyword import kwlist +from os import PathLike +from typing import ( + Any, + Dict, + Iterable, + Iterator, + List, + Literal, + NamedTuple, + Optional, + Tuple, + Union, + get_origin, +) + +import numpy as np +from numpy.typing import NDArray + +from flopy.mf6.utils.codegen.dfn import Dfn, DfnName, Metadata +from flopy.mf6.utils.codegen.ref import Refs +from flopy.mf6.utils.codegen.render import renderable +from flopy.mf6.utils.codegen.shim import SHIM +from flopy.mf6.utils.codegen.spec import Ref, Var, VarKind, Vars +from flopy.mf6.utils.codegen.utils import _try_get_type_name + +_SCALAR_TYPES = { + "keyword": bool, + "integer": int, + "double precision": float, + "string": str, +} +_NP_SCALAR_TYPES = { + "keyword": np.bool_, + "integer": np.int_, + "double precision": np.float64, + "string": np.str_, +} + + +class ContextName(NamedTuple): + """ + Uniquely identifies an input context by its name, which + consists of a <= 3-letter left term and optional right + term also of <= 3 letters. + + Notes + ----- + A single `DefinitionName` may be associated with one or + more `ContextName`s. For instance, a model DFN file will + produce both a NAM package class and also a model class. + + From the `ContextName` several other things are derived, + including: + + - the input context class' name + - a description of the context class + - the name of the source file to write + - the base class the context inherits from + + """ + + l: str + r: Optional[str] + + @property + def title(self) -> str: + """ + The input context's unique title. This is not + identical to `f"{l}{r}` in some cases, but it + remains unique. The title is substituted into + the file name and class name. + """ + + l, r = self + if self == ("sim", "nam"): + return "simulation" + if l is None: + return r + if r is None: + return l + if l == "sim": + return r + if l in ["sln", "exg"]: + return r + return f"{l}{r}" + + @property + def base(self) -> str: + """Base class from which the input context should inherit.""" + _, r = self + if self == ("sim", "nam"): + return "MFSimulationBase" + if r is None: + return "MFModel" + return "MFPackage" + + @property + def target(self) -> str: + """The source file name to generate.""" + return f"mf{self.title}.py" + + @property + def description(self) -> str: + """A description of the input context.""" + l, r = self + title = self.title.title() + if self.base == "MFPackage": + return f"Modflow{title} defines a {r.upper()} package." + elif self.base == "MFModel": + return f"Modflow{title} defines a {l.upper()} model." + elif self.base == "MFSimulationBase": + return """ + MFSimulation is used to load, build, and/or save a MODFLOW 6 simulation. + A MFSimulation object must be created before creating any of the MODFLOW 6 + model objects.""" + + +def get_context_names(dfn_name: DfnName) -> List[ContextName]: + """ + Returns a list of contexts this definition produces. + + Notes + ----- + An input definition may produce one or more input contexts. + + Model definition files produce both a model class context and + a model namefile package context. The same goes for simulation + definition files. All other definition files produce a single + context. + """ + if dfn_name.r == "nam": + if dfn_name.l == "sim": + return [ + ContextName(None, dfn_name.r), # nam pkg + ContextName(*dfn_name), # simulation + ] + else: + return [ + ContextName(*dfn_name), # nam pkg + ContextName(dfn_name.l, None), # model + ] + elif (dfn_name.l, dfn_name.r) in [ + ("gwf", "mvr"), + ("gwf", "gnc"), + ("gwt", "mvt"), + ]: + return [ContextName(*dfn_name), ContextName(None, dfn_name.r)] + return [ContextName(*dfn_name)] + + +@renderable(**SHIM) +@dataclass +class Context: + """ + An input context. Each of these is specified by a definition file + and becomes a generated class. A definition file may specify more + than one input context (e.g. model DFNs yield a model class and a + package class). + + Notes + ----- + A context class minimally consists of a name, a map of variables, + a map of records, and a list of metadata. + + The context class may inherit from a base class, and may specify + a parent context within which it can be created (the parent then + becomes the first `__init__` method parameter). + + A separate map of record variables is maintained because we will + generate named tuples for record types, and complex filtering of + e.g. nested maps of variables is awkward or impossible in Jinja. + TODO: make this a prerendering step + + """ + + name: ContextName + base: Optional[type] + parent: Optional[Union[type, str]] + description: Optional[str] + metadata: List[Metadata] + variables: Vars + records: Vars + references: Refs + + +def make_context( + name: ContextName, + dfn: Dfn, + common: Optional[Dfn] = None, + references: Optional[Refs] = None, +) -> Context: + """ + Extract a context descriptor from an input definition: + a structured representation of the input context that + can be used to generate an input data interface layer. + + Notes + ----- + Each input definition corresponds to a generated Python + source file. A definition may produce one or more input + context classes. + + A map of other definitions may be provided, in which case a + parameter in this context may act as kind of "foreign key", + identifying another context as a subpackage which this one + is related to. + """ + + common = common or dict() + references = references or dict() + ref = Ref.from_dfn(dfn) # this a ref? + refs = dict() # referenced contexts + records = dict() # record variables + + def _ntname(s): + """ + Convert a record name to the name of a corresponding named tuple. + + Notes + ----- + Dashes and underscores are removed, with title-casing for clauses + separated by them, and a trailing "record" is removed if present. + + """ + return ( + s.title().replace("record", "").replace("-", "_").replace("_", "") + ) + + def _parent() -> Optional[str]: + """ + Get the context's parent(s), i.e. context(s) which can + own an instance of this context. If this context is a + subpackage which can have multiple parent types, this + will be a Union of possible parent types, otherwise a + single parent type. + + Notes + ----- + We return a string directly instead of a type to avoid + the need to import `MFSimulation` in this file (avoids + potential for circular imports). + """ + l, r = dfn.name + if (l, r) == ("sim", "nam") and name == ("sim", "nam"): + return None + if l in ["sim", "exg", "sln"]: + return "MFSimulation" + if name.r is None: + return "MFSimulation" + if ref: + if len(ref.parents) > 1: + return f"Union[{', '.join([_try_get_type_name(t) for t in ref.parents])}]" + return ref.parents[0] + return "MFModel" + + parent = _parent() + + def _convert(var: Dict[str, Any], wrap: bool = False) -> Var: + """ + Transform a variable from its original representation in + an input definition to a specification suitable for type + hints, docstrings, an `__init__` method's signature, etc. + + This involves expanding nested type hierarchies, mapping + types to roughly equivalent Python primitives/composites, + and other shaping. + + The rules for optional variable defaults are as follows: + If a `default_value` is not provided, keywords are `False` + by default, everything else is `None`. + + If `wrap` is true, scalars will be wrapped as records with + keywords represented as string literals. This is useful for + unions, to distinguish between choices having the same type. + + Any filepath variable whose name functions as a foreign key + for another context will be given a pointer to the context. + + + Notes + ----- + This function does most of the work in the whole module. + A bit of a beast, and Codacy complains it's too complex, + but having it here allows using the outer function scope + (including the input definition, etc) without a bunch of + extra function parameters. And what it's doing is fairly + straightforward: map a variable specification from a DFN + into a corresponding Python representation. + + """ + + _name = var["name"] + _type = var.get("type", "unknown") + block = var.get("block", None) + shape = var.get("shape", None) + shape = None if shape == "" else shape + optional = var.get("optional", True) + in_record = var.get("in_record", False) + tagged = var.get("tagged", False) + description = var.get("description", "") + children = dict() + is_record = False + + def _description(descr: str) -> str: + """ + Make substitutions from common variable definitions, + remove backslashes, TODO: generate/insert citations. + """ + descr = descr.replace("\\", "") + _, replace, tail = descr.strip().partition("REPLACE") + if replace: + key, _, subs = tail.strip().partition(" ") + subs = literal_eval(subs) + cmn_var = common.get(key, None) + if cmn_var is None: + raise ValueError(f"Common variable not found: {key}") + descr = cmn_var.get("description", "") + if any(subs): + return descr.replace("\\", "").replace( + "{#1}", subs["{#1}"] + ) + return descr + return descr + + def _fields(record_name: str) -> Vars: + """Recursively load/convert a record's fields.""" + record = dfn[record_name] + field_names = record["type"].split()[1:] + fields: Dict[str, Var] = { + n: _convert(field, wrap=False) + for n, field in dfn.items() + if n in field_names + } + field_names = list(fields.keys()) + + # if the record represents a file... + if "file" in record_name: + # remove filein/fileout + for term in ["filein", "fileout"]: + if term in field_names: + fields.pop(term) + + # remove leading keyword + keyword = next(iter(fields), None) + if keyword: + fields.pop(keyword) + + # set the type + n = list(fields.keys())[0] + path_field = fields[n] + path_field._type = Union[str, PathLike] + fields[n] = path_field + + # if tagged, remove the leading keyword + elif record.get("tagged", False): + keyword = next(iter(fields), None) + if keyword: + fields.pop(keyword) + + return fields + + # go through all the possible input types + # from top (composite) to bottom (scalar): + # + # - list + # - union + # - record + # - array + # - scalar + + # list input, child is the item type + if _type.startswith("recarray"): + # make sure columns are defined + names = _type.split()[1:] + n_names = len(names) + if n_names < 1: + raise ValueError(f"Missing recarray definition: {_type}") + + # list input can have records or unions as rows. + # lists which have a consistent record type are + # regular, inconsistent record types irregular. + + # regular tabular/columnar data (1 record type) can be + # defined with a nested record (i.e. explicit) or with + # fields directly inside the recarray (implicit). list + # data for unions/keystrings necessarily comes nested. + + is_explicit_record = len(names) == 1 and dfn[names[0]][ + "type" + ].startswith("record") + + def _is_implicit_scalar_record(): + # if the record is defined implicitly and it has + # only scalar fields + types = [ + _try_get_type_name(v["type"]) + for n, v in dfn.items() + if n in names + ] + scalar_types = list(_SCALAR_TYPES.keys()) + return all(t in scalar_types for t in types) + + if is_explicit_record: + record_name = names[0] + record_spec = dfn[record_name] + record = _convert(record_spec, wrap=False) + children = {_ntname(record_name).lower(): record} + type_ = Iterable[record._type] + elif _is_implicit_scalar_record(): + record_name = _name + fields = _fields(record_name) + field_types = [f._type for f in fields.values()] + record_type = Tuple[tuple(field_types)] + record = Var( + name=record_name, + _type=record_type, + block=block, + children=fields, + description=description, + ) + records[_ntname(record_name)] = replace( + record, name=_ntname(record_name) + ) + record_type = namedtuple( + _ntname(record_name), + [_ntname(k) for k in fields.keys()], + ) + record = replace( + record, + _type=record_type, + name=_ntname(record_name).lower(), + ) + children = {_ntname(record_name): record} + type_ = Iterable[record_type] + else: + # implicit complex record (i.e. some fields are records or unions) + fields = {n: _convert(dfn[n], wrap=False) for n in names} + first = list(fields.values())[0] + single = len(fields) == 1 + record_name = first.name if single else _name + field_types = [f._type for f in fields.values()] + record_type = ( + first._type + if (single and get_origin(first._type) is Union) + else Tuple[tuple(field_types)] + ) + record = Var( + name=record_name, + _type=record_type, + block=block, + children=first.children if single else fields, + description=description, + ) + records[_ntname(record_name)] = replace( + record, name=_ntname(record_name) + ) + record_type = namedtuple( + _ntname(record_name), + [_ntname(k) for k in fields.keys()], + ) + record = replace( + record, + _type=record_type, + name=_ntname(record_name).lower(), + ) + children = {_ntname(record_name): record} + type_ = Iterable[record_type] + + # union (product), children are record choices + elif _type.startswith("keystring"): + names = _type.split()[1:] + children = {n: _convert(dfn[n], wrap=True) for n in names} + type_ = Union[tuple([c._type for c in children.values()])] + + # record (sum) type, children are fields + elif _type.startswith("record"): + children = _fields(_name) + if len(children) > 1: + record_type = Tuple[ + tuple([f._type for f in children.values()]) + ] + elif len(children) == 1: + t = list(children.values())[0]._type + # make sure we don't double-wrap tuples + record_type = t if get_origin(t) is tuple else Tuple[(t,)] + # TODO: if record has 1 field, accept value directly? + type_ = record_type + is_record = True + + # are we wrapping a var into a record + # as a choice in a union? if so use a + # string literal for the keyword e.g. + # `Tuple[Literal[...], T]` + elif wrap: + field_name = _name + field = _convert(var, wrap=False) + field_type = ( + Literal[field_name] if field._type is bool else field._type + ) + record_type = ( + Tuple[Literal[field_name]] + if field._type is bool + else Tuple[Literal[field_name], field._type] + ) + children = {field_name: replace(field, _type=field_type)} + type_ = record_type + is_record = True + + # at this point, if it has a shape, it's an array.. + # but if it's in a record make it a variadic tuple, + # and if its item type is a string use an iterable. + elif shape is not None: + scalars = list(_SCALAR_TYPES.keys()) + if in_record: + if _type not in scalars: + raise TypeError(f"Unsupported repeating type: {_type}") + type_ = Tuple[_SCALAR_TYPES[_type], ...] + elif _type in scalars and _SCALAR_TYPES[_type] is str: + type_ = Iterable[_SCALAR_TYPES[_type]] + else: + if _type not in _NP_SCALAR_TYPES.keys(): + raise TypeError(f"Unsupported array type: {_type}") + type_ = NDArray[_NP_SCALAR_TYPES[_type]] + + # finally a bog standard scalar + else: + # if it's a keyword, there are 2 cases where we want to convert + # it to a string literal: 1) it tags another variable, or 2) it + # is being wrapped into a record as a choice in a union + tag = _type == "keyword" and (tagged or wrap) + type_ = Literal[_name] if tag else _SCALAR_TYPES.get(_type, _type) + + # format the variable description + description = _description(description) + + # keywords default to False, everything else to None + default = var.get("default", False if type_ is bool else None) + if isinstance(default, str) and type_ is not str: + try: + default = literal_eval(default) + except: + pass + if _name in ["continue", "print_input"]: # hack... + default = None + + # if name is a reserved keyword, add a trailing underscore to it. + # convert dashes to underscores since it may become a class attr. + name_ = (f"{_name}_" if _name in kwlist else _name).replace("-", "_") + + # create var + var_ = Var( + name=name_, + _type=type_, + block=block, + description=description, + default=default, + children=children, + ) + + # if the var is a foreign key, register the referenced context + ref_ = references.get(_name, None) + if ref_: + var_.reference = ref_ + refs[_name] = ref_ + + # if the var is a record, make a named tuple for it + if is_record: + records[_ntname(name_)] = replace(var_, name=_ntname(name_)) + if any(children): + type_ = namedtuple( + _ntname(name_), [_ntname(k) for k in children.keys()] + ) + var_._type = type_ + + # wrap the var's type with Optional if it's optional + if optional: + var_._type = ( + Optional[type_] + if (type_ is not bool and not in_record and not wrap) + else type_ + ) + + return var_ + + def _variables() -> Vars: + """ + Return all input variables for an input context class. + + Notes + ----- + Not all variables become parameters; nested variables + will become components of composite parameters, e.g., + record fields, keystring (union) choices, list items. + + Variables may be added, depending on the context type. + """ + + vars_ = dfn.copy() + vars_ = { + name: _convert(var, wrap=False) + for name, var in vars_.items() + # filter composite components + # since we've already inflated + # their parents in the hierarchy + if not var.get("in_record", False) + } + + # set the name since we may have altered + # it when creating the variable (e.g. to + # avoid name/reserved keyword collisions) + return {v.name: v for v in vars_.values()} + + def _metadata() -> List[Metadata]: + """ + Get a list of the class' original definition attributes + as a partial, internal reproduction of the DFN contents. + + Notes + ----- + Currently, generated classes have a `.dfn` property that + reproduces the corresponding DFN sans a few attributes. + This represents the DFN in raw form, before adapting to + Python, consolidating nested types, etc. + """ + + def _fmt_var(var: Union[Var, List[Var]]) -> List[str]: + exclude = ["longname", "description"] + + def _fmt_name(k, v): + return v.replace("-", "_") if k == "name" else v + + return [ + " ".join([k, str(_fmt_name(k, v))]).strip() + for k, v in var.items() + if k not in exclude + ] + + meta = dfn.metadata or list() + return [["header"] + [m for m in meta]] + [ + _fmt_var(var) for var in dfn.omd.values(multi=True) + ] + + return Context( + name=name, + base=name.base, + parent=parent, + description=name.description, + metadata=_metadata(), + variables=_variables(), + records=records, + references=refs, + ) + + +def make_contexts( + dfn: Dfn, + common: Optional[Dfn] = None, + refs: Optional[Refs] = None, +) -> Iterator[Context]: + """Generate one or more input contexts from the given input definition.""" + for name in get_context_names(dfn.name): + yield make_context(name=name, dfn=dfn, common=common, references=refs) diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py new file mode 100644 index 0000000000..3b4478ccc8 --- /dev/null +++ b/flopy/mf6/utils/codegen/dfn.py @@ -0,0 +1,119 @@ +from collections import UserDict +from dataclasses import dataclass +from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Tuple + +from boltons.dictutils import OMD + + +class DfnName(NamedTuple): + """ + Uniquely identifies an input definition by its name, which + consists of a <= 3-letter left term and an optional right + term, also <= 3 letters. + + Notes + ----- + A single `DefinitionName` may be associated with one or + more `ContextName`s. For instance, a model DFN file will + produce both a NAM package class and also a model class. + """ + + l: str + r: str + + +Metadata = List[str] + + +@dataclass +class Dfn(UserDict): + """ + An MF6 input definition. + + Notes + ----- + Duplicate variable names are supported by an `OrderedMultiDict` + this class maintains alongside a `UserDict`-managed standard + dictionary; the former is retrievable with the `omd` property. + + This class should not be modified after loading. + """ + + name: Optional[DfnName] + metadata: Optional[Metadata] + + def __init__( + self, + variables: Iterable[Tuple[str, Dict[str, Any]]], + name: Optional[DfnName] = None, + metadata: Optional[Metadata] = None, + ): + self.omd = OMD(variables) + self.name = name + self.metadata = metadata + super().__init__(self.omd) + + +Dfns = Dict[str, Dfn] + + +def load_dfn(f, name: Optional[DfnName] = None) -> Dfn: + """ + Load an input definition from a definition file. + """ + + meta = None + vars_ = list() + var = dict() + + for line in f: + # remove whitespace/etc from the line + line = line.strip() + + # record context name and flopy metadata + # attributes, skip all other comment lines + if line.startswith("#"): + _, sep, tail = line.partition("flopy") + if sep == "flopy": + if meta is None: + meta = list() + tail = tail.strip() + if "solution_package" in tail: + tail = tail.split() + tail.pop(1) + meta.append(tail) + continue + _, sep, tail = line.partition("package-type") + if sep == "package-type": + if meta is None: + meta = list + meta.append(f"{sep} {tail.strip()}") + continue + _, sep, tail = line.partition("solution_package") + continue + + # if we hit a newline and the parameter dict + # is nonempty, we've reached the end of its + # block of attributes + if not any(line): + if any(var): + n = var["name"] + vars_.append((n, var)) + var = dict() + continue + + # split the attribute's key and value and + # store it in the parameter dictionary + key, _, value = line.partition(" ") + if key == "default_value": + key = "default" + if value in ["true", "false"]: + value = value == "true" + var[key] = value + + # add the final parameter + if any(var): + n = var["name"] + vars_.append((n, var)) + + return Dfn(variables=vars_, name=name, metadata=meta) diff --git a/flopy/mf6/utils/codegen/make.py b/flopy/mf6/utils/codegen/make.py new file mode 100644 index 0000000000..52a296a1ad --- /dev/null +++ b/flopy/mf6/utils/codegen/make.py @@ -0,0 +1,98 @@ +from pathlib import Path +from typing import ( + Optional, +) +from warnings import warn + +from jinja2 import Environment, PackageLoader +from modflow_devtools.misc import run_cmd + +# noqa: F401 +from flopy.mf6.utils.codegen.context import ( + get_context_names, + make_context, + make_contexts, +) +from flopy.mf6.utils.codegen.dfn import Dfn, DfnName, Dfns, load_dfn +from flopy.mf6.utils.codegen.ref import Ref, Refs + +_TEMPLATE_LOADER = PackageLoader("flopy", "mf6/utils/templates/") +_TEMPLATE_ENV = Environment(loader=_TEMPLATE_LOADER) +_TEMPLATE_NAME = "context.py.jinja" +_TEMPLATE = _TEMPLATE_ENV.get_template(_TEMPLATE_NAME) + + +def make_targets( + dfn: Dfn, + outdir: Path, + common: Optional[Dfn] = None, + refs: Optional[Refs] = None, + verbose: bool = False, +): + """Generate Python source file(s) from the given input definition.""" + + for context in make_contexts(dfn=dfn, common=common, refs=refs): + target = outdir / context.name.target + with open(target, "w") as f: + source = _TEMPLATE.render(**context.render()) + f.write(source) + if verbose: + print(f"Wrote {target}") + + +def make_all(dfndir: Path, outdir: Path, verbose: bool = False): + """Generate Python source files from the DFN files in the given location.""" + + # find definition files + paths = [ + p for p in dfndir.glob("*.dfn") if p.stem not in ["common", "flopy"] + ] + + # try to load common variables + common_path = dfndir / "common.dfn" + if not common_path.is_file: + warn("No common input definition file...") + common = None + else: + with open(common_path, "r") as f: + common = load_dfn(f) + + # load all the input definitions before we generate input + # contexts so we can create foreign key refs between them. + dfns: Dfns = {} + refs: Refs = {} + for p in paths: + name = DfnName(*p.stem.split("-")) + with open(p) as f: + dfn = load_dfn(f, name=name) + dfns[name] = dfn + ref = Ref.from_dfn(dfn) + if ref: + # key is the name of the file record + # that's the reference's foreign key + refs[ref.key] = ref + + # generate target files + for dfn in dfns.values(): + with open(p) as f: + make_targets( + dfn=dfn, + outdir=outdir, + refs=refs, + common=common, + verbose=verbose, + ) + + # generate __init__.py file + init_path = outdir / "__init__.py" + with open(init_path, "w") as f: + for dfn in dfns.values(): + for ctx in get_context_names(dfn.name): + prefix = "MF" if ctx.base == "MFSimulationBase" else "Modflow" + f.write( + f"from .mf{ctx.title} import {prefix}{ctx.title.title()}\n" + ) + + # format the generated files + run_cmd("ruff", "format", outdir, verbose=verbose) + run_cmd("ruff", "check", "--fix", outdir, verbose=True) diff --git a/flopy/mf6/utils/codegen/ref.py b/flopy/mf6/utils/codegen/ref.py new file mode 100644 index 0000000000..4162475a18 --- /dev/null +++ b/flopy/mf6/utils/codegen/ref.py @@ -0,0 +1,88 @@ +from dataclasses import dataclass +from typing import Dict, List, Optional, Union + +from flopy.mf6.utils.codegen.dfn import Dfn + + +@dataclass +class Ref: + """ + A foreign-key-like reference between a file input variable + and another input definition. This allows an input context + to refer to another input context, by including a filepath + variable whose name acts as a foreign key for a different + input context. Extra parameters are added to the referring + context's `__init__` method, so a selected "value" variable + defined in the referenced context can be provided directly + as an alternative to the file path (foreign key) variable. + + Parameters + ---------- + key : str + The name of the foreign key file input variable. + val : str + The name of the selected variable in the referenced context. + abbr : str + An abbreviation of the referenced context's name. + param : str + The subpackage parameter name. TODO: explain + parents : List[Union[str, type]] + The subpackage's supported parent types. + """ + + key: str + val: str + abbr: str + param: str + parents: List[Union[type, str]] + description: Optional[str] + + @classmethod + def from_dfn(cls, dfn: Dfn) -> Optional["Ref"]: + if not dfn.metadata: + return None + + lines = { + "subpkg": next( + iter( + m + for m in dfn.metadata + if isinstance(m, str) and m.startswith("subpac") + ), + None, + ), + "parent": next( + iter( + m + for m in dfn.metadata + if isinstance(m, str) and m.startswith("parent") + ), + None, + ), + } + + def _subpkg(): + line = lines["subpkg"] + _, key, abbr, param, val = line.split() + descr = dfn.get(val, dict()).get("description", None) + return { + "key": key, + "val": val, + "abbr": abbr, + "param": param, + "description": descr, + } + + def _parents(): + line = lines["parent"] + _, _, _type = line.split() + return _type.split("/") + + return ( + cls(**_subpkg(), parents=_parents()) + if all(v for v in lines.values()) + else None + ) + + +Refs = Dict[str, Ref] diff --git a/flopy/mf6/utils/codegen/render.py b/flopy/mf6/utils/codegen/render.py new file mode 100644 index 0000000000..953a26bdc2 --- /dev/null +++ b/flopy/mf6/utils/codegen/render.py @@ -0,0 +1,169 @@ +from dataclasses import asdict +from typing import Any, Callable, Dict, Iterable, Optional, Tuple, Union + +from flopy.mf6.utils.codegen.utils import ( + _try_get_enum_value, + _try_get_type_name, +) + +Predicate = Callable[[Any], bool] +Transform = Callable[[Any], Dict[str, str]] +Entry = Tuple[str, Any] +Entries = Iterable[Entry] + + +def renderable( + maybe_cls=None, + *, + add_entry: Optional[Iterable[Tuple[Predicate, Entries]]] = None, + keep_none: Optional[Iterable[str]] = None, + quote_str: Optional[Iterable[str]] = None, + transform: Optional[Iterable[Tuple[Predicate, Transform]]] = None, + type_name: Optional[Iterable[str]] = None, +): + """ + Decorator for dataclasses which are meant + to be passed into a Jinja template. The + decorator adds a `.render()` method to + the decorated class, which recursively + converts the instance to a dictionary + with (by default) the `asdict()` builtin + `dataclasses` module function, plus a + few modifications to make the instance + easier to work with from the template. + + By default, attributes with value `None` + are dropped before conversion to a `dict`. + To specify that a given attribute should + remain even with a `None` value, use the + `keep_none` parameter. + + When a string value is to become the RHS + of an assignment or an argument-passing + expression, it needs to be wrapped with + quotation marks before insertion into + the template. To indicate an attribute's + value should be wrapped with quotation + marks, use the `quote_str` parameter. + + Straightforward stringification of `type` + doesn't always give a suitable result for + use within a template; `type_name` can be + used to specify attributes whose value is + a `type` that needs conversion to a more + template-friendly string. + + Finally, arbitrary transformations can be + specified with the `transform` parameter, + which accepts a set of predicate/function + pairs; see below for more information on + how to use the transformation mechanism. + + Notes + ----- + Jinja supports attribute- and dictionary- + based access on arbitrary objects but does + not support arbitrary expressions, and has + only a limited set of custom filters. This + can make it awkward to express some things. + + This decorator is intended as a convenient + way to modify dataclass instances to make + them more palatable for templates. It also + aims to keep keep edge cases incidental to + the current design of MF6 input framework + cleanly isolated from the reimplementation + of which this code is a part. + + The `dataclasses` module provides a builtin + `asdict()` function to recursively convert + a nested object hierarchy to a dictionary; + this function has a `dict_factory` function + parameter which can be used to change how a + `dict` is constructed from each instance of + a dataclass found within the root instance. + + The basic idea behind this decorator is for + the developer to specify conditions in which + a given dataclass instance should be altered, + and a function to make the alteration. These + are provided as a collection of `Predicate`/ + `Transform` pairs. + + Transformations might be for convenience, or + to handle special cases where an object has + some other need for modification. Edge cases + in the MF6 compatibility layer (for example, + some of the logic in `mfstructure.py` which + determines the members of generated classes) + can be isolated as rendering transformations. + This allows keeping more general templating + infrastructure free of incidental complexity + while we move toward a leaner core framework. + + Because a transformation function accepts an + instance of a dataclass and converts it to a + dictionary, only one transformation function + can be applied per dataclass instance. Where + multiple predicates evaluate to true for the + instance, only the first is applied. + + """ + + add_entry = add_entry or list() + quote_str = quote_str or list() + keep_none = keep_none or list() + transform = transform or list() + type_name = type_name or list() + + def __renderable(cls): + def _render(d: dict) -> dict: + def _render_val(k, v): + v = _try_get_enum_value(v) + if k in type_name: + v = _try_get_type_name(v) + if k in quote_str and isinstance(v, str): + v = f'"{v}"' + return v + + # drop nones except where requested to keep them + return { + k: _render_val(k, v) + for k, v in d.items() + if (k in keep_none or v is not None) + } + + def _dict(o): + # apply the first transform with a matching predicate + d = dict(o) + for p, t in transform: + if p(o): + d = t(o) + + for p, e in add_entry: + if not p(d): + continue + if e is None: + raise ValueError(f"No value for entry {k}") + for k, v in e: + if callable(v): + v = v(d) + d[k] = v + + return d + + def render(self) -> dict: + """ + Recursively render the dataclass instance. + """ + return _render( + asdict(self, dict_factory=lambda o: _render(_dict(o))) + ) + + setattr(cls, "render", render) + return cls + + # first arg value depends on the decorator usage: + # class if `@renderable`, `None` if `@renderable()`. + # referenced from https://github.com/python-attrs/attrs/blob/a59c5d7292228dfec5480388b5f6a14ecdf0626c/src/attr/_next_gen.py#L405C4-L406C65 + return __renderable if maybe_cls is None else __renderable(maybe_cls) diff --git a/flopy/mf6/utils/codegen/shim.py b/flopy/mf6/utils/codegen/shim.py new file mode 100644 index 0000000000..3c68c7e663 --- /dev/null +++ b/flopy/mf6/utils/codegen/shim.py @@ -0,0 +1,472 @@ +import os +from os import PathLike +from typing import Iterable, Optional, Union, get_args, get_origin + +from numpy.typing import ArrayLike +from pandas import DataFrame + +from flopy.mf6.utils.codegen.spec import Var, VarKind + + +def _add_exg_vars(ctx): + """ + Add initializer parameters for an exchange input context. + Exchanges need different parameters than a typical package. + """ + d = dict(ctx) + a = d["name"].r[:3] + b = d["name"].r[:3] + default = f"{a.upper()}6-{b.upper()}6" + vars_ = d["variables"].copy() + vars_ = { + "parent": Var( + name="parent", + _type="MFSimulation", + description=( + "Simulation that this package is a part of. " + "Package is automatically added to simulation " + "when it is initialized." + ), + ), + "loading_package": Var( + name="loading_package", + _type=bool, + description=( + "Do not set this parameter. It is intended for " + "debugging and internal processing purposes only." + ), + default=False, + ), + "exgtype": Var( + name="exgtype", + _type=str, + default=default, + description="The exchange type.", + ), + "exgmnamea": Var( + name="exgmnamea", + _type=str, + description="The name of the first model in the exchange.", + ), + "exgmnameb": Var( + name="exgmnameb", + _type=str, + description="The name of the second model in the exchange.", + ), + **vars_, + "filename": Var( + name="filename", + _type=Union[str, PathLike], + description="File name for this package.", + ), + "pname": Var( + name="pname", + _type=str, + description="Package name for this package.", + ), + } + + if d["references"]: + for key, ref in d["references"].items(): + if key not in vars_: + continue + vars_[ref["val"]] = Var( + name=ref["val"], + description=ref.get("description", None), + reference=ref, + ) + + d["variables"] = vars_ + return d + + +def _add_pkg_vars(ctx): + """Add variables for a package context.""" + d = dict(ctx) + parent_name = "parent" + vars_ = d["variables"].copy() + vars_ = { + parent_name: Var( + name=parent_name, + _type=d["parent"], + description="Parent that this package is part of.", + ), + "loading_package": Var( + name="loading_package", + _type=bool, + description=( + "Do not set this variable. It is intended for debugging " + "and internal processing purposes only." + ), + default=False, + ), + **vars_, + "filename": Var( + name="filename", + _type=str, + description="File name for this package.", + ), + "pname": Var( + name="pname", + _type=str, + description="Package name for this package.", + ), + } + + if d["name"].l == "utl": + vars_["parent_file"] = Var( + name="parent_file", + _type=Union[str, PathLike], + description=( + "Parent package file that references this package. Only needed " + "for utility packages (mfutl*). For example, mfutllaktab package " + "must have a mfgwflak package parent_file." + ), + ) + + if d["references"] and d["name"] != (None, "nam"): + for key, ref in d["references"].items(): + if key not in vars_: + continue + vars_[ref["val"]] = Var( + name=ref["val"], + description=ref.get("description", None), + reference=ref, + ) + + d["variables"] = vars_ + return d + + +def _add_mdl_vars(ctx): + """Add variables for a model context.""" + d = dict(ctx) + vars_ = d["variables"].copy() + vars_ = { + "simulation": Var( + name="simulation", + _type="MFSimulation", + description=( + "Simulation that this model is part of. " + "Model is automatically added to the simulation " + "when it is initialized." + ), + ), + "modelname": Var( + name="modelname", + _type=str, + description="The name of the model.", + default="model", + ), + "model_nam_file": Var( + name="model_nam_file", + _type=Optional[Union[str, PathLike]], + description=( + "The relative path to the model name file from model working folder." + ), + ), + "version": Var( + name="version", + _type=str, + description="The version of modflow", + default="mf6", + ), + "exe_name": Var( + name="exe_name", + _type=str, + description="The executable name.", + default="mf6", + ), + "model_rel_path": Var( + name="model_ws", + _type=Union[str, PathLike], + description="The model working folder path.", + default=os.curdir, + ), + **vars_, + } + + if d["references"]: + for key, ref in d["references"].items(): + if key not in vars_: + continue + vars_[ref["val"]] = Var( + name=ref["val"], + description=ref.get("description", None), + reference=ref, + ) + + d["variables"] = vars_ + return d + + +def _add_sim_vars(ctx): + """Add variables for a simulation context.""" + d = dict(ctx) + vars_ = d["variables"].copy() + skip_init = [ + "tdis6", + "models", + "exchanges", + "mxiter", + "solutiongroup", + ] + for k in skip_init: + var = vars_.get(k, None) + if var: + var["init_param"] = False + vars_[k] = var + vars_ = { + "sim_name": Var( + name="sim_name", + _type=str, + default="sim", + description="Name of the simulation.", + ), + "version": Var( + name="version", + _type=str, + default="mf6", + ), + "exe_name": Var( + name="exe_name", + _type=Union[str, PathLike], + default="mf6", + ), + "sim_ws": Var( + name="sim_ws", + _type=Union[str, PathLike], + default=os.curdir, + ), + "verbosity_level": Var( + name="verbosity_level", + _type=int, + default=1, + ), + "write_headers": Var( + name="write_headers", + _type=bool, + default=True, + ), + "use_pandas": Var( + name="use_pandas", + _type=bool, + default=True, + ), + "lazy_io": Var( + name="lazy_io", + _type=bool, + default=False, + ), + **vars_, + } + + if d["references"]: + for key, ref in d["references"].items(): + if key not in vars_: + continue + vars_[ref["val"]] = Var( + name=ref["val"], + description=ref.get("description", None), + reference=ref, + ) + + d["variables"] = vars_ + return d + + +def _add_ctx_vars(o): + d = dict(o) + if d["name"].base == "MFSimulationBase": + return _add_sim_vars(d) + elif d["name"].base == "MFModel": + return _add_mdl_vars(d) + elif d["name"].base == "MFPackage": + if d["name"].l == "exg": + return _add_exg_vars(d) + else: + return _add_pkg_vars(d) + return d + + +def _is_ctx(o) -> bool: + d = dict(o) + return "name" in d and "base" in d + + +def _is_var(o) -> bool: + d = dict(o) + return "name" in d and "_type" in d + + +def _init_param(o) -> bool: + """Whether the var is an `__init__` method parameter.""" + d = dict(o) + if d["name"] in [ + "packages", + "tdis6", + "models", + "exchanges", + "mxiter", + "solutiongroup", + ]: + return False + if d.get("ref", None): + return False + return True + + +def _init_assign(o) -> bool: + """ + Whether to assign arguments to self in the + `__init__` method. if this is false, assume + the template has conditionals for any more + involved initialization needs. + """ + d = dict(o) + return d["name"] in ["exgtype", "exgnamea", "exgnameb"] + + +def _init_build(o) -> bool: + """ + Whether to call `build_mfdata()` on the variable. + in the `__init__` method. + """ + d = dict(o) + ref = d.get("ref", None) + if ref: + return False + if d["name"] in [ + "parent", + "loading_package", + "exgtype", + "exgnamea", + "exgnameb", + "filename", + "pname", + "parent_file" "simulation", + "modelname", + "model_nam_file", + "version", + "exe_name", + "model_rel_path", + "sim_name", + "sim_ws", + "verbosity_level", + "write_headers", + "use_pandas", + "lazy_io", + ]: + return False + return True + + +def _init_super(o) -> bool: + """ + Whether to pass the variable to `super().__init__()` + by name in the `__init__` method.""" + d = dict(o) + return d["name"] in [ + "parent", + "loading_package", + "filename", + "pname", + "simulation", + "modelname", + "model_nam_file", + "version", + "exe_name", + "model_rel_path", + "sim_name", + "sim_ws", + "verbosity_level", + "write_headers", + "use_pandas", + "lazy_io", + ] + + +def _class_attr(o) -> bool: + """Whether to add a class attribute for the variable.""" + d = dict(o) + if d.get("ref", None): + return True + kind = VarKind.from_type(d["_type"]) + if kind != VarKind.Scalar: + return True + return False + + +def _kind(o) -> VarKind: + # the variable's general shape. because Jinja + # doesn't allow arbitrary expressions, and it + # doesn't seem to have a subclass test filter, + # we need this for template conditional exprs. + d = dict(o) + return VarKind.from_type(d["_type"]) + + +def _loose_type(o) -> type: + """ + Derive a "loose" (lenient) typing attribute + from the variable's type, which can be more + accepting than the variable's specification. + Used for init method params, while the spec + itself (in e.g. the class docstring) can be + the more descriptive (i.e. unmodified) type. + """ + d = dict(o) + if d["kind"] == VarKind.Array: + # arrays can be described as NDArray with a + # type parameter, or ndarray with type and + # shape parameters, while init params can + # be specified more loosely as ArrayLike. + return ArrayLike + if d["kind"] == VarKind.List: + # lists can be iterables regardless whether + # regular. if regular then accept dataframe + _iterable = Iterable[get_args(d["_type"])[0]] + children = list(d["children"].values()) + if ( + any(children) + and VarKind.from_type(children[0]["_type"]) == VarKind.Union + ): + return _iterable + return Union[_iterable, DataFrame] + # TODO transient lists: + # map of lists by stress period, or... + # iterable appled to all stress periods + return d["_type"] + + +SHIM = { + "keep_none": ["default", "block"], + "quote_str": ["default"], + "type_name": ["_type"], + "transform": [ + # context-specific parameters + # for the `__init__()` method. + # do it as a `transform` (not + # `add_entry`) so we are able + # to control the param order. + (_is_ctx, _add_ctx_vars) + ], + "add_entry": [ + ( + _is_var, + [ + ("kind", _kind), + ("loose_type", _loose_type), + ("init_param", _init_param), + ("init_assign", _init_assign), + ("init_build", _init_build), + ("init_super", _init_super), + ("class_attr", _class_attr), + ], + ), + ], +} +""" +Arguments for `renderable` as applied to `Context` +to support the current `flopy.mf6` input framework. +""" diff --git a/flopy/mf6/utils/codegen/spec.py b/flopy/mf6/utils/codegen/spec.py new file mode 100644 index 0000000000..21253ef8ca --- /dev/null +++ b/flopy/mf6/utils/codegen/spec.py @@ -0,0 +1,88 @@ +import collections +from dataclasses import dataclass +from enum import Enum +from typing import Any, Dict, Optional, Union, get_args, get_origin + +import numpy as np +from numpy.typing import ArrayLike, NDArray + +from flopy.mf6.utils.codegen.dfn import Metadata +from flopy.mf6.utils.codegen.ref import Ref + + +class VarKind(Enum): + """ + An input variable's kind. This is an enumeration + of the general shapes of data MODFLOW 6 accepts, + convertible to/from Python primitives/composites. + """ + + Array = "array" + Scalar = "scalar" + Record = "record" + Union = "union" + List = "list" + + @classmethod + def from_type(cls, t: type) -> Optional["VarKind"]: + origin = get_origin(t) + args = get_args(t) + if origin is Union: + if len(args) >= 2 and args[-1] is type(None): + if len(args) > 2: + return VarKind.Union + return cls.from_type(args[0]) + return VarKind.Union + if origin is np.ndarray or origin is NDArray or origin is ArrayLike: + return VarKind.Array + elif origin is collections.abc.Iterable or origin is list: + return VarKind.List + elif origin is tuple: + return VarKind.Record + try: + if issubclass(t, (bool, int, float, str)): + return VarKind.Scalar + except: + pass + return None + + def to_type(self) -> type: + # TODO + pass + + +@dataclass +class Var: + """An input variable specification.""" + + name: str + _type: Union[type, str] + block: Optional[str] + description: Optional[str] + default: Optional[Any] + children: Optional[Dict[str, "Var"]] + metadata: Optional[Metadata] + reference: Optional[Ref] + + def __init__( + self, + name: str, + _type: Optional[type] = None, + block: Optional[str] = None, + description: Optional[str] = None, + default: Optional[Any] = None, + children: Optional["Vars"] = None, + metadata: Optional[Metadata] = None, + reference: Optional[Ref] = None, + ): + self.name = name + self._type = _type or Any + self.block = block + self.description = description + self.default = default + self.children = children + self.metadata = metadata + self.reference = reference + + +Vars = Dict[str, Var] diff --git a/flopy/mf6/utils/codegen/utils.py b/flopy/mf6/utils/codegen/utils.py new file mode 100644 index 0000000000..db659f147a --- /dev/null +++ b/flopy/mf6/utils/codegen/utils.py @@ -0,0 +1,41 @@ +import collections +from enum import Enum +from typing import Any, ForwardRef, Literal, Union, get_args, get_origin + +import numpy as np + + +def _try_get_type_name(t) -> str: + """Convert a type to a name suitable for templating.""" + origin = get_origin(t) + args = get_args(t) + if origin is Literal: + args = ['"' + a + '"' for a in args] + return f"Literal[{', '.join(args)}]" + elif origin is Union: + if len(args) >= 2 and args[-1] is type(None): + if len(args) > 2: + return f"Optional[Union[{', '.join([_try_get_type_name(a) for a in args[:-1]])}]]" + return f"Optional[{_try_get_type_name(args[0])}]" + return f"Union[{', '.join([_try_get_type_name(a) for a in args])}]" + elif origin is tuple: + return f"Tuple[{', '.join([_try_get_type_name(a) for a in args])}]" + elif origin is collections.abc.Iterable: + return f"Iterable[{', '.join([_try_get_type_name(a) for a in args])}]" + elif origin is list: + return f"List[{', '.join([_try_get_type_name(a) for a in args])}]" + elif origin is np.ndarray: + return f"NDArray[np.{_try_get_type_name(args[1].__args__[0])}]" + elif origin is np.dtype: + return str(t) + elif isinstance(t, ForwardRef): + return t.__forward_arg__ + elif t is Ellipsis: + return "..." + elif isinstance(t, type): + return t.__qualname__ + return t + + +def _try_get_enum_value(v: Any) -> Any: + return v.value if isinstance(v, Enum) else v diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index dcf9fca42c..6d76aa8cf6 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -81,1657 +81,9 @@ """ -import collections -import os -from ast import literal_eval -from collections import UserDict, namedtuple -from dataclasses import asdict, dataclass, replace -from enum import Enum -from keyword import kwlist -from os import PathLike from pathlib import Path -from typing import ( - Any, - Callable, - Dict, - ForwardRef, - Iterable, - Iterator, - List, - Literal, - NamedTuple, - Optional, - Tuple, - Union, - get_args, - get_origin, -) -from warnings import warn - -import numpy as np -from boltons.dictutils import OMD -from jinja2 import Environment, PackageLoader -from modflow_devtools.misc import run_cmd -from numpy.typing import ArrayLike, NDArray - - -def _try_get_type_name(t) -> str: - """Convert a type to a name suitable for templating.""" - origin = get_origin(t) - args = get_args(t) - if origin is Literal: - args = ['"' + a + '"' for a in args] - return f"Literal[{', '.join(args)}]" - elif origin is Union: - if len(args) >= 2 and args[-1] is type(None): - if len(args) > 2: - return f"Optional[Union[{', '.join([_try_get_type_name(a) for a in args[:-1]])}]]" - return f"Optional[{_try_get_type_name(args[0])}]" - return f"Union[{', '.join([_try_get_type_name(a) for a in args])}]" - elif origin is tuple: - return f"Tuple[{', '.join([_try_get_type_name(a) for a in args])}]" - elif origin is collections.abc.Iterable: - return f"Iterable[{', '.join([_try_get_type_name(a) for a in args])}]" - elif origin is list: - return f"List[{', '.join([_try_get_type_name(a) for a in args])}]" - elif origin is np.ndarray: - return f"NDArray[np.{_try_get_type_name(args[1].__args__[0])}]" - elif origin is np.dtype: - return str(t) - elif isinstance(t, ForwardRef): - return t.__forward_arg__ - elif t is Ellipsis: - return "..." - elif isinstance(t, type): - return t.__qualname__ - return t - - -def _try_get_enum_value(v: Any) -> Any: - return v.value if isinstance(v, Enum) else v - - -def renderable( - maybe_cls=None, - *, - wrap_str: Optional[List[str]] = None, - keep_none: Optional[List[str]] = None, - transform: Optional[Dict[str, Callable[[Any], Any]]] = None, -): - """ - An object meant to be passed into a template - as a "rendered" dictionary, where "rendering" - means transforming key/value pairs to a form - appropriate for use within the template. - - Notes - ----- - Transformations might be for convenience* or - to handle special cases where a variable has - edge cases or other need for alteration**. - - *Jinja supports attribute- and dictionary- - based access but no arbitrary expressions, - and only a limited set of custom filters. - This can make it awkward to express some - things. - - **This is convenient for handling complexity - incidental to the current mf6 data framework. - Transforming values at render time helps to - isolate special cases from the more general - templating infrastructure, so the framework - can be refactored more easily over time. - - The object *must* be a dataclass. - - Common use cases include: - - converting types to suitably qualified type names - - optionally removing key/value pairs whose value is None - - optionally quoting strings forming the RHS of an assignment or - argument passing expression - - """ - - wrap_str = wrap_str or list() - keep_none = keep_none or list() - - def __renderable(cls): - def _render(d: dict) -> dict: - def _render_key(k): - return k - - def _render_val(k, v): - v = _try_get_type_name(_try_get_enum_value(v)) - - def noop(v): - return v - - return (transform.get(k, noop) if transform else noop)(v) - - # drop nones except where keep requested - _d = { - _render_key(k): _render_val(k, v) - for k, v in d.items() - if (k in keep_none or v is not None) - } - - # wrap string values where requested - for k in wrap_str: - v = _d.get(k, None) - if v is not None and isinstance(v, str): - _d[k] = f'"{v}"' - - return _d - - def render(self) -> dict: - """ - Recursively render the dataclass instance. - """ - return _render( - asdict(self, dict_factory=lambda d: _render(dict(d))) - ) - - setattr(cls, "render", render) - return cls - - # first arg value depends on the decorator usage: - # class if `@renderable`, `None` if `@renderable()`. - # referenced from https://github.com/python-attrs/attrs/blob/a59c5d7292228dfec5480388b5f6a14ecdf0626c/src/attr/_next_gen.py#L405C4-L406C65 - return __renderable if maybe_cls is None else __renderable(maybe_cls) - - -class ContextName(NamedTuple): - """ - Uniquely identifies an input context by its name, which - consists of a <= 3-letter left term and optional right - term also of <= 3 letters. - - Notes - ----- - A single `DefinitionName` may be associated with one or - more `ContextName`s. For instance, a model DFN file will - produce both a NAM package class and also a model class. - - From the `ContextName` several other things are derived, - including: - - - the input context class' name - - a description of the context class - - the name of the source file to write - - the base class the context inherits from - - """ - - l: str - r: Optional[str] - - @property - def title(self) -> str: - """ - The input context's unique title. This is not - identical to `f"{l}{r}` in some cases, but it - remains unique. The title is substituted into - the file name and class name. - """ - - l, r = self - if self == ("sim", "nam"): - return "simulation" - if l is None: - return r - if r is None: - return l - if l == "sim": - return r - if l in ["sln", "exg"]: - return r - return f"{l}{r}" - - @property - def base(self) -> str: - """Base class from which the input context should inherit.""" - _, r = self - if self == ("sim", "nam"): - return "MFSimulationBase" - if r is None: - return "MFModel" - return "MFPackage" - - @property - def target(self) -> str: - """The source file name to generate.""" - return f"mf{self.title}.py" - - @property - def description(self) -> str: - """A description of the input context.""" - l, r = self - title = self.title.title() - if self.base == "MFPackage": - return f"Modflow{title} defines a {r.upper()} package." - elif self.base == "MFModel": - return f"Modflow{title} defines a {l.upper()} model." - elif self.base == "MFSimulationBase": - return """ - MFSimulation is used to load, build, and/or save a MODFLOW 6 simulation. - A MFSimulation object must be created before creating any of the MODFLOW 6 - model objects.""" - - -class DfnName(NamedTuple): - """ - Uniquely identifies an input definition by its name, which - consists of a <= 3-letter left term and an optional right - term, also <= 3 letters. - - Notes - ----- - A single `DefinitionName` may be associated with one or - more `ContextName`s. For instance, a model DFN file will - produce both a NAM package class and also a model class. - """ - - l: str - r: str - - @property - def contexts(self) -> List[ContextName]: - """ - Returns a list of contexts this definition will produce. - - Notes - ----- - Model definition files produce both a model class context and - a model namefile package context. The same goes for simulation - definition files. All other definition files produce a single - context. - """ - if self.r == "nam": - if self.l == "sim": - return [ - ContextName(None, self.r), # nam pkg - ContextName(*self), # simulation - ] - else: - return [ - ContextName(*self), # nam pkg - ContextName(self.l, None), # model - ] - elif (self.l, self.r) in [ - ("gwf", "mvr"), - ("gwf", "gnc"), - ("gwt", "mvt"), - ]: - return [ContextName(*self), ContextName(None, self.r)] - return [ContextName(*self)] - - -Metadata = List[str] - - -@dataclass -class Dfn(UserDict): - """ - An MF6 input definition. - """ - - name: Optional[DfnName] - metadata: Optional[Metadata] - - def __init__( - self, - variables: Iterable[Tuple[str, Dict[str, Any]]], - name: Optional[DfnName] = None, - metadata: Optional[Metadata] = None, - ): - self.omd = OMD(variables) - self.name = name - self.metadata = metadata - super().__init__(self.omd) - - -Dfns = Dict[str, Dfn] - - -def load_dfn(f, name: Optional[DfnName] = None) -> Dfn: - """ - Load an input definition from a definition file. - """ - - meta = None - vars_ = list() - var = dict() - - for line in f: - # remove whitespace/etc from the line - line = line.strip() - - # record context name and flopy metadata - # attributes, skip all other comment lines - if line.startswith("#"): - _, sep, tail = line.partition("flopy") - if sep == "flopy": - if meta is None: - meta = list() - tail = tail.strip() - if "solution_package" in tail: - tail = tail.split() - tail.pop(1) - meta.append(tail) - continue - _, sep, tail = line.partition("package-type") - if sep == "package-type": - if meta is None: - meta = list - meta.append(f"{sep} {tail.strip()}") - continue - _, sep, tail = line.partition("solution_package") - continue - - # if we hit a newline and the parameter dict - # is nonempty, we've reached the end of its - # block of attributes - if not any(line): - if any(var): - n = var["name"] - vars_.append((n, var)) - var = dict() - continue - - # split the attribute's key and value and - # store it in the parameter dictionary - key, _, value = line.partition(" ") - if key == "default_value": - key = "default" - if value in ["true", "false"]: - value = value == "true" - var[key] = value - - # add the final parameter - if any(var): - n = var["name"] - vars_.append((n, var)) - - return Dfn(variables=vars_, name=name, metadata=meta) - - -@dataclass -class Subpkg: - """ - A foreign-key-like reference between a file input variable - and a subpackage definition. This allows an input context - to reference a subpackage by including a variable with an - appropriate name. - - Parameters - ---------- - key : str - The name of the file input variable identifying the - referenced subpackage. - val : str - The name of the variable containing subpackage data - in the referenced subpackage. - abbr : str - An abbreviation of the subpackage's name. - param : str - The subpackage parameter name. TODO: explain - parents : List[type] - The subpackage's supported parent types. - """ - - key: str - val: str - abbr: str - param: str - parents: List[Union[type, str]] - description: Optional[str] - - @classmethod - def from_dfn(cls, dfn: Dfn) -> Optional["Subpkg"]: - if not dfn.metadata: - return None - - lines = { - "subpkg": next( - iter( - m - for m in dfn.metadata - if isinstance(m, str) and m.startswith("subpac") - ), - None, - ), - "parent": next( - iter( - m - for m in dfn.metadata - if isinstance(m, str) and m.startswith("parent") - ), - None, - ), - } - - def _subpkg(): - line = lines["subpkg"] - _, key, abbr, param, val = line.split() - descr = dfn.get(val, dict()).get("description", None) - return { - "key": key, - "val": val, - "abbr": abbr, - "param": param, - "description": descr, - } - - def _parents(): - line = lines["parent"] - _, _, _type = line.split() - return _type.split("/") - - return ( - cls(**_subpkg(), parents=_parents()) - if all(v for v in lines.values()) - else None - ) - - -Subpkgs = Dict[str, Subpkg] - - -class VarKind(Enum): - """ - An input variable's kind. This is an enumeration - of the general shapes of data MODFLOW 6 accepts. - """ - - Array = "array" - Scalar = "scalar" - Record = "record" - Union = "union" - List = "list" - - @classmethod - def from_type(cls, t: type) -> Optional["VarKind"]: - origin = get_origin(t) - args = get_args(t) - if origin is Union: - if len(args) >= 2 and args[-1] is type(None): - if len(args) > 2: - return VarKind.Union - return cls.from_type(args[0]) - return VarKind.Union - if origin is np.ndarray or origin is NDArray or origin is ArrayLike: - return VarKind.Array - elif origin is collections.abc.Iterable or origin is list: - return VarKind.List - elif origin is tuple: - return VarKind.Record - try: - if issubclass(t, (bool, int, float, str)): - return VarKind.Scalar - except: - pass - return None - - -@dataclass -class Var: - """A variable in a MODFLOW 6 input context.""" - - name: str - _type: Union[type, str] - block: Optional[str] - description: Optional[str] - default: Optional[Any] - children: Optional[Dict[str, "Var"]] - meta: Optional[List[str]] - subpkg: Optional[Subpkg] - kind: Optional[VarKind] - is_choice: bool = False - init_param: bool = True - init_assign: bool = False - init_build: bool = False - init_super: bool = False - class_attr: bool = False - - def __init__( - self, - name: str, - _type: Optional[type] = None, - block: Optional[str] = None, - description: Optional[str] = None, - default: Optional[Any] = None, - parent: Optional["Var"] = None, - children: Optional["Vars"] = None, - meta: Optional[Metadata] = None, - subpkg: Optional[Subpkg] = None, - kind: Optional[VarKind] = None, - is_choice: bool = False, - init_param: bool = True, - init_assign: bool = False, - init_build: bool = False, - init_super: bool = False, - class_attr: bool = False, - ): - self.name = name - self._type = _type or Any - self.block = block - self.description = description - self.default = default - self.parent = parent - self.children = children - self.meta = meta - self.subpkg = subpkg - # TODO: the rest of the attributes below are - # needed to handle complexities in the input - # context classes; in a future version, they - # will ideally not be necessary. - # --- - # the variable's general kind. - # this is ofc derivable on demand but Jinja - # doesn't allow arbitrary expressions, and it - # doesn't seem to have `subclass`-ish filters. - self.kind = kind or VarKind.from_type(_type) - # similarly, whether the variable is a choice - # in a union. this is derivable from .parent, - # but awkward to do in Jinja, so use a flag. - self.is_choice = is_choice - # whether the var is an init method parameter - self.init_param = init_param - # whether to assign arguments to self in the - # init method body. if this is false, assume - # the template has conditionals for any more - # involved initialization needs. - self.init_assign = init_assign - # whether to call `build_mfdata()` to build - # the parameter. - self.init_build = init_build - # whether to pass arg to super().__init__() - self.init_super = init_super - # whether the variable has a corresponding - # class attribute - self.class_attr = class_attr - - -Vars = Dict[str, Var] - - -@renderable( - wrap_str=["default"], - keep_none=["block", "default"], - # TODO replace the flags on Var with transforms? -) -@dataclass -class Context: - """ - An input context. Each of these is specified by a definition file - and becomes a generated class. A definition file may specify more - than one input context (e.g. model DFNs yield a model class and a - package class). - - Notes - ----- - A context class minimally consists of a name, a map of variables, - a map of records, and a list of metadata. - - A separate map of record variables is maintained because we will - generate named tuples for record types, and complex filtering of - e.g. nested maps of variables is awkward or impossible in Jinja. - - The context class may inherit from a base class, and may specify - a parent context within which it can be created (the parent then - becomes the first `__init__` method parameter). - - """ - - name: ContextName - base: Optional[type] - parent: Optional[Union[type, str]] - description: Optional[str] - metadata: Metadata - variables: Vars - records: Vars - subpkg: bool - - -_SCALAR_TYPES = { - "keyword": bool, - "integer": int, - "double precision": float, - "string": str, -} -_NP_SCALAR_TYPES = { - "keyword": np.bool_, - "integer": np.int_, - "double precision": np.float64, - "string": np.str_, -} - - -def make_context( - name: ContextName, - dfn: Dfn, - common: Optional[Dfn] = None, - subpkgs: Optional[Subpkgs] = None, -) -> Context: - """ - Convert an MF6 input definition to a structured descriptor - of an input context class to create with a Jinja template. - - Notes - ----- - Each input definition corresponds to a generated Python - source file. A definition may produce one or more input - context classes. - - A map of other definitions may be provided, in which case a - parameter in this context may act as kind of "foreign key", - identifying another context as a subpackage which this one - is related to. - """ - - common = common or dict() - subpkgs = subpkgs or dict() - _subpkg = Subpkg.from_dfn(dfn) - records = dict() - - def _nt_name(s, trims=False): - """ - Convert a record name to the name of a corresponding named tuple. - - Notes - ----- - Dashes and underscores are removed, with title-casing for clauses - separated by them, and a trailing "record" is removed if present. - - """ - s = s.title().replace("record", "").replace("-", "_").replace("_", "") - if trims: - s = s[:-1] if s.endswith("s") else s - return s - - def _parent() -> Optional[str]: - """ - Get the context's parent(s), i.e. context(s) which can - own an instance of this context. If this context is a - subpackage which can have multiple parent types, this - will be a Union of possible parent types, otherwise a - single parent type. - - Notes - ----- - We return a string directly instead of a type to avoid - the need to import `MFSimulation` in this file (avoids - potential for circular imports). - """ - l, r = dfn.name - if (l, r) == ("sim", "nam") and name == ("sim", "nam"): - return None - if l in ["sim", "exg", "sln"]: - return "MFSimulation" - if name.r is None: - return "MFSimulation" - if _subpkg: - if len(_subpkg.parents) > 1: - return f"Union[{', '.join([_try_get_type_name(t) for t in _subpkg.parents])}]" - return _subpkg.parents[0] - return "MFModel" - - parent = _parent() - - def _convert( - var: Dict[str, Any], - wrap: bool = False, - ) -> Var: - """ - Transform a variable from its original representation in - an input definition to a specification suitable for type - hints, docstrings, an `__init__` method's signature, etc. - - This involves expanding nested type hierarchies, mapping - types to roughly equivalent Python primitives/composites, - and other shaping. - - Notes - ----- - The rules for optional variable defaults are as follows: - If a `default_value` is not provided, keywords are `False` - by default, everything else is `None`. - - If `wrap` is true, scalars will be wrapped as records with - keywords represented as string literals. This is useful for - unions, to distinguish between choices having the same type. - - Any variable whose name functions as a key for a subpackage - will be provided with a subpackage reference. - """ - - # var attributes to be converted - _name = var["name"] - _type = var.get("type", "unknown") - block = var.get("block", None) - shape = var.get("shape", None) - shape = None if shape == "" else shape - optional = var.get("optional", True) - in_record = var.get("in_record", False) - tagged = var.get("tagged", False) - description = var.get("description", "") - children = None - is_record = False - class_attr = var.get("class_attr", False) - init_build = var.get("init_build", True) - - def _description(descr: str) -> str: - """ - Make substitutions from common variable definitions, - remove backslashes, generate/insert citations, etc. - TODO: insert citations. - """ - descr = descr.replace("\\", "") - _, replace, tail = descr.strip().partition("REPLACE") - if replace: - key, _, replacements = tail.strip().partition(" ") - replacements = literal_eval(replacements) - common_var = common.get(key, None) - if common_var is None: - raise ValueError(f"Common variable not found: {key}") - descr = common_var.get("description", "") - if any(replacements): - return descr.replace("\\", "").replace( - "{#1}", replacements["{#1}"] - ) - return descr - return descr - - def _fields(record_name: str) -> Vars: - """ - Recursively load/convert a record's fields. - - Notes - ----- - This function is provided because records - need extra processing; we remove keywords - and 'filein'/'fileout', which are details - of the mf6io format, not of python/flopy. - """ - record = dfn[record_name] - field_names = record["type"].split()[1:] - fields: Dict[str, Var] = { - n: _convert(field, wrap=False) - for n, field in dfn.items() - if n in field_names - } - field_names = list(fields.keys()) - - # if the record represents a file... - if "file" in record_name: - # remove filein/fileout - for term in ["filein", "fileout"]: - if term in field_names: - fields.pop(term) - - # remove leading keyword - keyword = next(iter(fields), None) - if keyword: - fields.pop(keyword) - - # set the type - n = list(fields.keys())[0] - path_field = fields[n] - path_field._type = Union[str, os.PathLike] - fields[n] = path_field - - # if tagged, remove the leading keyword - elif record.get("tagged", False): - keyword = next(iter(fields), None) - if keyword: - fields.pop(keyword) - - return fields - - # go through all the possible input types - # from top (composite) to bottom (scalar): - # - # - list - # - union - # - record - # - array - # - scalar - # - # list input can have records or unions as rows. - # lists which have a consistent record type are - # regular, inconsistent record types irregular. - if _type.startswith("recarray"): - # flag as a class attribute (ListTemplateGenerator etc) - class_attr = var.get("class_attr", True) - - # make sure columns are defined - names = _type.split()[1:] - n_names = len(names) - if n_names < 1: - raise ValueError(f"Missing recarray definition: {_type}") - - # regular tabular/columnar data (1 record type) can be - # defined with a nested record (i.e. explicit) or with - # fields directly inside the recarray (implicit). list - # data for unions/keystrings necessarily comes nested. - - is_explicit_record = len(names) == 1 and dfn[names[0]][ - "type" - ].startswith("record") - - def _is_implicit_scalar_record(): - # if the record is defined implicitly and it has - # only scalar fields - types = [ - _try_get_type_name(v["type"]) - for n, v in dfn.items() - if n in names - ] - scalar_types = list(_SCALAR_TYPES.keys()) - return all(t in scalar_types for t in types) - - if is_explicit_record: - record_name = names[0] - record_spec = dfn[record_name] - record_type = _convert(record_spec, wrap=False) - children = {_nt_name(record_name).lower(): record_type} - type_ = Iterable[record_type._type] - elif _is_implicit_scalar_record(): - record_name = _name - record_fields = _fields(record_name) - field_types = [f._type for f in record_fields.values()] - record_type = Tuple[tuple(field_types)] - record = Var( - name=record_name, - _type=record_type, - block=block, - children=record_fields, - description=description, - ) - records[_nt_name(record_name, trims=True)] = replace( - record, name=_nt_name(record_name, trims=True) - ) - record_type = namedtuple( - _nt_name(record_name, trims=True), - [_nt_name(k) for k in record_fields.keys()], - ) - record = replace( - record, - _type=record_type, - name=_nt_name(record_name, trims=True).lower(), - ) - children = {_nt_name(record_name, trims=True): record} - type_ = Iterable[record_type] - else: - # implicit complex record (i.e. some fields are records or unions) - record_fields = { - n: _convert(dfn[n], wrap=False) for n in names - } - first = list(record_fields.values())[0] - single = len(record_fields) == 1 - record_name = first.name if single else _name - _t = [f._type for f in record_fields.values()] - record_type = ( - first._type - if (single and first.kind == VarKind.Union) - else Tuple[tuple(_t)] - ) - record = Var( - name=record_name, - _type=record_type, - block=block, - children=first.children if single else record_fields, - description=description, - ) - records[_nt_name(record_name)] = replace( - record, name=_nt_name(record_name) - ) - record_type = namedtuple( - _nt_name(record_name), - [_nt_name(k) for k in record_fields.keys()], - ) - record = replace( - record, - _type=record_type, - name=_nt_name(record_name).lower(), - ) - type_ = Iterable[record_type] - - # union (product), children are record choices - elif _type.startswith("keystring"): - # flag as a class attribute (ListTemplateGenerator etc) - class_attr = var.get("class_attr", True) - - names = _type.split()[1:] - children = {n: _convert(dfn[n], wrap=True) for n in names} - type_ = Union[tuple([c._type for c in children.values()])] - - # record (sum) type, children are fields - elif _type.startswith("record"): - # flag as a class attribute (ListTemplateGenerator etc) - class_attr = var.get("class_attr", True) - - children = _fields(_name) - if len(children) > 1: - record_type = Tuple[ - tuple([f._type for f in children.values()]) - ] - elif len(children) == 1: - t = list(children.values())[0]._type - # make sure we don't double-wrap tuples - record_type = t if get_origin(t) is tuple else Tuple[(t,)] - # TODO: if record has 1 field, accept value directly? - type_ = record_type - is_record = True - - # are we wrapping a var into a record - # as a choice in a union? if so use a - # string literal for the keyword e.g. - # `Tuple[Literal[...], T]` - elif wrap: - field_name = _name - field = _convert(var, wrap=False) - field_type = ( - Literal[field_name] if field._type is bool else field._type - ) - record_type = ( - Tuple[Literal[field_name]] - if field._type is bool - else Tuple[Literal[field_name], field._type] - ) - children = { - field_name: replace(field, _type=field_type, is_choice=True) - } - type_ = record_type - is_record = True - - # at this point, if it has a shape, it's an array.. - # but if it's in a record make it a variadic tuple, - # and if its item type is a string use an iterable. - elif shape is not None: - # flag as a class attribute (ListTemplateGenerator etc) - class_attr = var.get("class_attr", True) - scalars = list(_SCALAR_TYPES.keys()) - if in_record: - if _type not in scalars: - raise TypeError(f"Unsupported repeating type: {_type}") - type_ = Tuple[_SCALAR_TYPES[_type], ...] - elif _type in scalars and _SCALAR_TYPES[_type] is str: - type_ = Iterable[_SCALAR_TYPES[_type]] - else: - if _type not in _NP_SCALAR_TYPES.keys(): - raise TypeError(f"Unsupported array type: {_type}") - type_ = NDArray[_NP_SCALAR_TYPES[_type]] - - # finally a bog standard scalar - else: - # if it's a keyword, there are 2 cases where we want to convert - # it to a string literal: 1) it tags another variable, or 2) it - # is being wrapped into a record as a choice in a union - tag = _type == "keyword" and (tagged or wrap) - type_ = Literal[_name] if tag else _SCALAR_TYPES.get(_type, _type) - - # format the variable description - description = _description(description) - - # keywords default to False, everything else to None - default = var.get("default", False if type_ is bool else None) - if isinstance(default, str) and type_ is not str: - try: - default = literal_eval(default) - except: - pass - if _name in ["continue", "print_input"]: # hack... - default = None - - # if name is a reserved keyword, add a trailing underscore to it. - # convert dashes to underscores since it may become a class attr. - name_ = (f"{_name}_" if _name in kwlist else _name).replace("-", "_") - - # create var - var_ = Var( - name=name_, - _type=type_, - block=block, - description=description, - default=default, - children=children, - init_param=True, - init_build=init_build, - class_attr=class_attr, - ) - - # check if the variable references a subpackage - subpkg = subpkgs.get(_name, None) - if subpkg: - var_.init_build = False - var_.subpkg = subpkg - - # if this is a record, make a named tuple for it - if is_record: - records[_nt_name(name_)] = replace(var_, name=_nt_name(name_)) - if children: - type_ = namedtuple( - _nt_name(name_), [_nt_name(k) for k in children.keys()] - ) - var_._type = type_ - - # make optional if needed - if optional: - var_._type = ( - Optional[type_] - if (type_ is not bool and not in_record and not wrap) - else type_ - ) - - return var_ - - def _variables() -> Vars: - """ - Return all input variables for an input context class. - - Notes - ----- - Not all variables become parameters; nested variables - will become components of composite parameters, e.g., - record fields, keystring (union) choices, list items. - - Variables may be added, depending on the context type. - """ - - vars_ = dfn.copy() - vars_ = { - name: _convert(var, wrap=False) - for name, var in vars_.items() - # filter composite components - # since we've already inflated - # their parents in the hierarchy - if not var.get("in_record", False) - } - - # set the name since we may have altered - # it when creating the variable (e.g. to - # avoid name/reserved keyword collisions. - vars_ = {v.name: v for v in vars_.values()} - - def _add_exg_vars(_vars: Vars) -> Vars: - """ - Add initializer parameters for an exchange context. - Exchanges need different parameters than a typical - package. - """ - a = name.r[:3] - b = name.r[:3] - default = f"{a.upper()}6-{b.upper()}6" - vars_ = { - "parent": Var( - name="parent", - _type="MFSimulation", - description=( - "Simulation that this package is a part of. " - "Package is automatically added to simulation " - "when it is initialized." - ), - init_param=True, - init_assign=False, - init_build=False, - init_super=True, - ), - "loading_package": Var( - name="loading_package", - _type=bool, - description=( - "Do not set this parameter. It is intended for " - "debugging and internal processing purposes only." - ), - default=False, - init_param=True, - init_assign=False, - init_build=False, - init_super=True, - ), - "exgtype": Var( - name="exgtype", - _type=str, - default=default, - description="The exchange type.", - init_param=True, - init_assign=True, - init_build=False, - init_super=False, - ), - "exgmnamea": Var( - name="exgmnamea", - _type=str, - description="The name of the first model in the exchange.", - init_param=True, - init_assign=True, - init_super=False, - ), - "exgmnameb": Var( - name="exgmnameb", - _type=str, - description="The name of the second model in the exchange.", - init_param=True, - init_assign=True, - init_build=False, - init_super=False, - ), - **_vars, - "filename": Var( - name="filename", - _type=Union[str, PathLike], - description="File name for this package.", - init_param=True, - init_assign=False, - init_build=False, - init_super=True, - ), - "pname": Var( - name="pname", - _type=str, - description="Package name for this package.", - init_param=True, - init_assign=False, - init_build=False, - init_super=True, - ), - } - - # if a reference map is provided, - # find any variables referring to - # subpackages, and attach another - # "value" variable for them all.. - # allows passing data directly to - # `__init__` instead of a path to - # load the subpackage from. maybe - # impossible if the data variable - # doesn't appear in the reference - # definition, though. - if subpkgs: - for k, subpkg in subpkgs.items(): - key = vars_.get(k, None) - if not key: - continue - vars_[subpkg.key].init_param = False - vars_[subpkg.key].init_build = True - vars_[subpkg.key].class_attr = True - vars_[subpkg.val] = Var( - name=subpkg.val, - description=subpkg.description, - subpkg=subpkg, - init_param=True, - init_assign=False, - init_super=False, - init_build=False, - ) - - return vars_ - - def _add_pkg_vars(_vars: Vars) -> Vars: - """Add variables for a package context.""" - parent_name = "parent" - vars_ = { - parent_name: Var( - name=parent_name, - _type=parent, - description="Parent that this package is part of.", - init_param=True, - init_assign=False, - init_super=True, - ), - "loading_package": Var( - name="loading_package", - _type=bool, - description=( - "Do not set this variable. It is intended for debugging " - "and internal processing purposes only." - ), - default=False, - init_param=True, - init_assign=False, - init_super=True, - ), - **_vars, - "filename": Var( - name="filename", - _type=str, - description="File name for this package.", - init_param=True, - init_assign=False, - init_super=True, - ), - "pname": Var( - name="pname", - _type=str, - description="Package name for this package.", - init_param=True, - init_assign=False, - init_super=True, - ), - } - - # if context is a subpackage add - # a `parent_file` variable which - # is the path to the subpackage's - # parent context - subpkg = Subpkg.from_dfn(dfn) - if subpkg and dfn.name.l == "utl": - vars_["parent_file"] = Var( - name="parent_file", - _type=Union[str, PathLike], - description=( - "Parent package file that references this package. Only needed " - "for utility packages (mfutl*). For example, mfutllaktab package " - "must have a mfgwflak package parent_file." - ), - init_param=True, - init_assign=False, - ) - - # if a reference map is provided, - # find any variables referring to - # subpackages, and attach another - # "value" variable for them all.. - # allows passing data directly to - # `__init__` instead of a path to - # load the subpackage from. maybe - # impossible if the data variable - # doesn't appear in the reference - # definition, though. - if subpkgs and name != (None, "nam"): - for k, subpkg in subpkgs.items(): - key = vars_.get(k, None) - if not key: - continue - vars_[subpkg.key].init_param = False - vars_[subpkg.key].init_build = True - vars_[subpkg.key].class_attr = True - vars_[subpkg.val] = Var( - name=subpkg.val, - description=subpkg.description, - subpkg=subpkg, - init_param=True, - init_assign=False, - init_super=False, - init_build=False, - ) - - return vars_ - - def _add_mdl_vars(_vars: Vars) -> Vars: - """Add variables for a model context.""" - vars_ = _vars.copy() - packages = _vars.get("packages", None) - if packages: - packages.init_param = False - vars_["packages"] = packages - - vars_ = { - "simulation": Var( - name="simulation", - _type="MFSimulation", - description=( - "Simulation that this model is part of. " - "Model is automatically added to the simulation " - "when it is initialized." - ), - init_param=True, - init_assign=False, - init_super=True, - ), - "modelname": Var( - name="modelname", - _type=str, - description="The name of the model.", - default="model", - init_param=True, - init_assign=False, - init_super=True, - ), - "model_nam_file": Var( - name="model_nam_file", - _type=Optional[Union[str, PathLike]], - description=( - "The relative path to the model name file from model working folder." - ), - init_param=True, - init_assign=False, - init_super=True, - ), - "version": Var( - name="version", - _type=str, - description="The version of modflow", - default="mf6", - init_param=True, - init_assign=False, - init_super=True, - ), - "exe_name": Var( - name="exe_name", - _type=str, - description="The executable name.", - default="mf6", - init_param=True, - init_assign=False, - init_super=True, - ), - "model_rel_path": Var( - name="model_ws", - _type=Union[str, PathLike], - description="The model working folder path.", - default=os.curdir, - init_param=True, - init_assign=False, - init_super=True, - ), - **vars_, - } - - # if a reference map is provided, - # find any variables referring to - # subpackages, and attach another - # "value" variable for them all.. - # allows passing data directly to - # `__init__` instead of a path to - # load the subpackage from. maybe - # impossible if the data variable - # doesn't appear in the reference - # definition, though. - if subpkgs: - for k, subpkg in subpkgs.items(): - key = vars_.get(k, None) - if not key: - continue - vars_[subpkg.key].init_param = False - vars_[subpkg.key].class_attr = True - vars_[subpkg.val] = Var( - name=subpkg.val, - description=subpkg.description, - subpkg=subpkg, - init_param=True, - init_assign=False, - init_super=False, - init_build=False, - ) - - return vars_ - - def _add_sim_params(_vars: Vars) -> Vars: - """Add variables for a simulation context.""" - vars_ = _vars.copy() - skip_init = [ - "tdis6", - "models", - "exchanges", - "mxiter", - "solutiongroup", - ] - for k in skip_init: - var = vars_.get(k, None) - if var: - var.init_param = False - vars_[k] = var - vars_ = { - "sim_name": Var( - name="sim_name", - _type=str, - default="sim", - description="Name of the simulation.", - init_param=True, - init_assign=False, - init_super=True, - ), - "version": Var( - name="version", - _type=str, - default="mf6", - init_param=True, - init_assign=False, - init_super=True, - ), - "exe_name": Var( - name="exe_name", - _type=Union[str, PathLike], - default="mf6", - init_param=True, - init_assign=False, - init_super=True, - ), - "sim_ws": Var( - name="sim_ws", - _type=Union[str, PathLike], - default=os.curdir, - init_param=True, - init_assign=False, - init_super=True, - ), - "verbosity_level": Var( - name="verbosity_level", - _type=int, - default=1, - init_param=True, - init_assign=False, - init_super=True, - ), - "write_headers": Var( - name="write_headers", - _type=bool, - default=True, - init_param=True, - init_assign=False, - init_super=True, - ), - "use_pandas": Var( - name="use_pandas", - _type=bool, - default=True, - init_param=True, - init_assign=False, - init_super=True, - ), - "lazy_io": Var( - name="lazy_io", - _type=bool, - default=False, - init_param=True, - init_assign=False, - init_super=True, - ), - **vars_, - } - - # if a reference map is provided, - # find any variables referring to - # subpackages, and attach another - # "value" variable for them all.. - # allows passing data directly to - # `__init__` instead of a path to - # load the subpackage from. maybe - # impossible if the data variable - # doesn't appear in the reference - # definition, though. - if subpkgs: - for k, subpkg in subpkgs.items(): - key = vars_.get(k, None) - if not key: - continue - vars_[subpkg.key].init_param = False - vars_[subpkg.key].init_build = False - vars_[subpkg.key].class_attr = True - vars_[subpkg.param] = Var( - name=subpkg.param, - description=subpkg.description, - subpkg=subpkg, - init_param=True, - init_assign=False, - init_super=False, - init_build=False, - ) - - return vars_ - - # add initializer method parameters - # for this particular context type - if name.base == "MFSimulationBase": - vars_ = _add_sim_params(vars_) - elif name.base == "MFModel": - vars_ = _add_mdl_vars(vars_) - elif name.base == "MFPackage": - if name.l == "exg": - vars_ = _add_exg_vars(vars_) - else: - vars_ = _add_pkg_vars(vars_) - - return vars_ - - def _metadata() -> List[Metadata]: - """ - Get a list of the class' original definition attributes - as a partial, internal reproduction of the DFN contents. - - Notes - ----- - Currently, generated classes have a `.dfn` property that - reproduces the corresponding DFN sans a few attributes. - This represents the DFN in raw form, before adapting to - Python, consolidating nested types, etc. - """ - - def _fmt_var(var: Union[Var, List[Var]]) -> List[str]: - exclude = ["longname", "description"] - - def _fmt_name(k, v): - return v.replace("-", "_") if k == "name" else v - - return [ - " ".join([k, str(_fmt_name(k, v))]).strip() - for k, v in var.items() - if k not in exclude - ] - - meta = dfn.metadata or list() - return [["header"] + [m for m in meta]] + [ - _fmt_var(var) for var in dfn.omd.values(multi=True) - ] - - return Context( - name=name, - base=name.base, - parent=parent, - description=name.description, - metadata=_metadata(), - variables=_variables(), - records=records, - subpkg=bool(_subpkg), - ) - - -def make_contexts( - dfn: Dfn, - common: Optional[Dfn] = None, - subpkgs: Optional[Subpkgs] = None, -) -> Iterator[Context]: - for name in dfn.name.contexts: - yield make_context(name=name, dfn=dfn, common=common, subpkgs=subpkgs) - - -_TEMPLATE_ENV = Environment( - loader=PackageLoader("flopy", "mf6/utils/templates/"), -) - - -def make_targets( - dfn: Dfn, - outdir: Path, - common: Optional[Dfn] = None, - subpkgs: Optional[Subpkgs] = None, - verbose: bool = False, -): - """ - Generate Python source file(s) from the given input definition. - - Notes - ----- - - Model definitions will produce two files / classes, one for the - model itself and one for its corresponding control file package. - - All other definitions currently produce a single file and class. - """ - - template = _TEMPLATE_ENV.get_template("context.py.jinja") - for context in make_contexts(dfn=dfn, common=common, subpkgs=subpkgs): - target = outdir / context.name.target - with open(target, "w") as f: - source = template.render(**context.render()) - f.write(source) - if verbose: - print(f"Wrote {target}") - - -def make_all(dfndir: Path, outdir: Path, verbose: bool = False): - """Generate Python source files from the DFN files in the given location.""" - - # find definition files - paths = [ - p for p in dfndir.glob("*.dfn") if p.stem not in ["common", "flopy"] - ] - - # try to load common variables - common_path = dfndir / "common.dfn" - if not common_path.is_file: - warn("No common input definition file...") - common = None - else: - with open(common_path, "r") as f: - common = load_dfn(f) - - # load all definitions first before we generate targets, - # so we can identify subpackages and create references - # between package/subpackage contexts. - dfns = dict() - subpkgs = dict() - for p in paths: - name = DfnName(*p.stem.split("-")) - with open(p) as f: - dfn = load_dfn(f, name=name) - dfns[name] = dfn - subpkg = Subpkg.from_dfn(dfn) - if subpkg: - # key is the name of the file record - # that corresponds to the subpackage - subpkgs[subpkg.key] = subpkg - - # generate target files - for dfn in dfns.values(): - with open(p) as f: - make_targets( - dfn=dfn, - outdir=outdir, - subpkgs=subpkgs, - common=common, - verbose=verbose, - ) - - # write __init__.py file - init_path = outdir / "__init__.py" - with open(init_path, "w") as f: - for dfn in dfns.values(): - for context in dfn.name.contexts: - prefix = ( - "MF" if context.base == "MFSimulationBase" else "Modflow" - ) - f.write( - f"from .mf{context.title} import {prefix}{context.title.title()}\n" - ) - - # format the generated files - run_cmd("ruff", "format", outdir, verbose=verbose) - run_cmd("ruff", "check", "--fix", outdir, verbose=True) +from flopy.mf6.utils.codegen.make import make_all _MF6_PATH = Path(__file__).parents[1] _DFN_PATH = _MF6_PATH / "data" / "dfn" diff --git a/flopy/mf6/utils/templates/docstring_params.jinja b/flopy/mf6/utils/templates/docstring_params.jinja index 3aeea087c6..f914c758c7 100644 --- a/flopy/mf6/utils/templates/docstring_params.jinja +++ b/flopy/mf6/utils/templates/docstring_params.jinja @@ -1,9 +1,9 @@ {%- for v in variables.values() recursive %} * {{ v.name }} : {{ v._type }} -{%- if v.description is defined and not v.is_choice %} +{%- if v.description is defined and v.description is not none %} {{ v.description|wordwrap|indent(4 + (loop.depth * 4), first=True) }} {%- endif %} -{%- if v.children is defined and loop.depth < 2-%} +{%- if v.children is defined and v.children is not none and loop.depth < 2 -%} {{ loop(v.children.values())|indent(4) }} {%- endif %} {% endfor -%} \ No newline at end of file diff --git a/flopy/mf6/utils/templates/init.jinja b/flopy/mf6/utils/templates/init.jinja index 7aca7f7082..5b9be805cd 100644 --- a/flopy/mf6/utils/templates/init.jinja +++ b/flopy/mf6/utils/templates/init.jinja @@ -30,10 +30,10 @@ def __init__( self.name_file.{{ n }}.set_data({{ n }}) self.{{ n }} = self.name_file.{{ n }} {% endif -%} - {%- if var.subpkg is defined and base != "MFModel" %} - self.{{ var.subpkg.param }} = self._create_package( - "{{ var.subpkg.abbr }}", - {{ var.subpkg.param }} + {%- if var.reference is defined and base != "MFModel" %} + self.{{ var.reference.param }} = self._create_package( + "{{ var.reference.abbr }}", + {{ var.reference.param }} ) {% endif -%} {% endfor -%} @@ -48,14 +48,14 @@ def __init__( {% for n, var in variables.items() -%} {%- if var.init_assign -%} self.{{ n }} = {{ n }} - {% elif var.subpkg is defined and var.init_build -%} + {% elif var.reference is defined and var.init_build -%} self._{{ n }} = self.build_mfdata("{{ n }}", None) - {% elif var.subpkg is defined and name.r != "nam" -%} - self._{{ var.subpkg.abbr }}_package = self.build_child_package( - "{{ var.subpkg.abbr }}", - {{ var.subpkg.val }}, - "{{ var.subpkg.param }}", - self._{{ var.subpkg.key }} + {% elif var.reference is defined and name.r != "nam" -%} + self._{{ var.reference.abbr }}_package = self.build_child_package( + "{{ var.reference.abbr }}", + {{ var.reference.val }}, + "{{ var.reference.param }}", + self._{{ var.reference.key }} ) {% elif var.init_build -%} self.{{ n }} = self.build_mfdata("{% if n == "continue_" %}continue{% else %}{{ n }}{% endif %}", {{ n }}) diff --git a/flopy/mf6/utils/templates/records.jinja b/flopy/mf6/utils/templates/records.jinja index cb67bcefe6..2ca88eb8ef 100644 --- a/flopy/mf6/utils/templates/records.jinja +++ b/flopy/mf6/utils/templates/records.jinja @@ -6,7 +6,7 @@ {%- endif %} {%- for v in var.children.values() recursive %} * {{ v.name }} : {{ v._type }} -{%- if v.description is defined and not v.is_choice %} +{%- if v.description is defined and v.description is not none %} {{ v.description|wordwrap|indent(4 + (loop.depth * 4), first=True) }} {%- endif %} {%- if v.children is defined -%} diff --git a/pyproject.toml b/pyproject.toml index 5ed2c83362..c4d566b94a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,8 @@ dependencies = [ "numpy>=1.20.3", "matplotlib >=1.4.0", "pandas >=2.0.0", + # necessary for createpackages.py + "ruff" ] dynamic = ["version", "readme"] @@ -41,7 +43,7 @@ dynamic = ["version", "readme"] dev = ["flopy[lint,test,optional,doc]"] lint = [ "cffconvert", - "ruff" + # "ruff" ] test = [ "flopy[lint]", From 6a0dfacbf31fe0223c7ada6c769834eec403ca0e Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 8 Oct 2024 06:56:29 -0400 Subject: [PATCH 46/46] continuing module reorg --- autotest/test_codegen.py | 35 ++--- autotest/test_dfn.py | 22 +++ flopy/mf6/utils/codegen/context.py | 120 +++++++--------- flopy/mf6/utils/codegen/dfn.py | 153 ++++++++++++--------- flopy/mf6/utils/codegen/make.py | 10 +- flopy/mf6/utils/codegen/render.py | 6 +- flopy/mf6/utils/codegen/shim.py | 60 ++++++-- flopy/mf6/utils/templates/attrs.jinja | 2 +- flopy/mf6/utils/templates/context.py.jinja | 2 +- 9 files changed, 231 insertions(+), 179 deletions(-) create mode 100644 autotest/test_dfn.py diff --git a/autotest/test_codegen.py b/autotest/test_codegen.py index 1e76c5d393..f4ef3773c1 100644 --- a/autotest/test_codegen.py +++ b/autotest/test_codegen.py @@ -1,8 +1,8 @@ +import traceback from ast import Assign, ClassDef, expr from ast import parse as parse_ast from pprint import pformat from shutil import copytree -import traceback from typing import List, Union from warnings import warn @@ -11,9 +11,9 @@ from autotest.conftest import get_project_root_path from flopy.mf6.utils.codegen.context import get_context_names +from flopy.mf6.utils.codegen.dfn import Dfn from flopy.mf6.utils.codegen.make import ( DfnName, - load_dfn, make_all, make_context, make_contexts, @@ -31,37 +31,30 @@ ] -@pytest.mark.parametrize("dfn_name", DFN_NAMES) -def test_load_dfn(dfn_name): - dfn_path = DFN_PATH / f"{dfn_name}.dfn" - with open(dfn_path, "r") as f: - dfn = load_dfn(f, name=DfnName(*dfn_name.split("-"))) - - @pytest.mark.parametrize( - "dfn_name, n_flat, n_params", [("gwf-ic", 2, 2), ("prt-prp", 40, 18)] + "dfn, n_flat, n_params", [("gwf-ic", 2, 2), ("prt-prp", 40, 18)] ) -def test_make_context(dfn_name, n_flat, n_params): +def test_make_context(dfn, n_flat, n_params): with open(DFN_PATH / "common.dfn") as f: - common = load_dfn(f) + commonvars = Dfn.load(f) - with open(DFN_PATH / f"{dfn_name}.dfn") as f: - dfn_name = DfnName(*dfn_name.split("-")) - dfn = load_dfn(f, name=dfn_name) + with open(DFN_PATH / f"{dfn}.dfn") as f: + dfn = DfnName(*dfn.split("-")) + definition = Dfn.load(f, name=dfn) - context_names = get_context_names(dfn_name) + context_names = get_context_names(dfn) context_name = context_names[0] - context = make_context(context_name, dfn, common=common) + context = make_context(context_name, definition, commonvars) assert len(context_names) == 1 assert len(context.variables) == n_params - assert len(context.metadata) == n_flat + 1 # +1 for metadata + assert len(context.definition.metadata) == n_flat + 1 # +1 for metadata @pytest.mark.skip(reason="TODO") @pytest.mark.parametrize("dfn_name", ["gwf-ic", "prt-prp", "gwf-nam"]) def test_make_contexts(dfn_name): with open(DFN_PATH / "common.dfn") as f: - common = load_dfn(f) + common = Dfn.load(f) # TODO @@ -69,11 +62,11 @@ def test_make_contexts(dfn_name): @pytest.mark.parametrize("dfn_name", DFN_NAMES) def test_make_targets(dfn_name, function_tmpdir): with open(DFN_PATH / "common.dfn") as f: - common = load_dfn(f) + common = Dfn.load(f) with open(DFN_PATH / f"{dfn_name}.dfn", "r") as f: dfn_name = DfnName(*dfn_name.split("-")) - dfn = load_dfn(f, name=dfn_name) + dfn = Dfn.load(f, name=dfn_name) make_targets(dfn, function_tmpdir, common=common) for ctx_name in get_context_names(dfn_name): diff --git a/autotest/test_dfn.py b/autotest/test_dfn.py new file mode 100644 index 0000000000..1899457177 --- /dev/null +++ b/autotest/test_dfn.py @@ -0,0 +1,22 @@ +import pytest + +from autotest.conftest import get_project_root_path +from flopy.mf6.utils.codegen.dfn import Dfn +from flopy.mf6.utils.codegen.make import DfnName + +PROJ_ROOT = get_project_root_path() +MF6_PATH = PROJ_ROOT / "flopy" / "mf6" +TGT_PATH = MF6_PATH / "modflow" +DFN_PATH = MF6_PATH / "data" / "dfn" +DFN_NAMES = [ + dfn.stem + for dfn in DFN_PATH.glob("*.dfn") + if dfn.stem not in ["common", "flopy"] +] + + +@pytest.mark.parametrize("dfn_name", DFN_NAMES) +def test_load_dfn(dfn_name): + dfn_path = DFN_PATH / f"{dfn_name}.dfn" + with open(dfn_path, "r") as f: + dfn = Dfn.load(f, name=DfnName(*dfn_name.split("-"))) diff --git a/flopy/mf6/utils/codegen/context.py b/flopy/mf6/utils/codegen/context.py index 77041d68ea..da783345aa 100644 --- a/flopy/mf6/utils/codegen/context.py +++ b/flopy/mf6/utils/codegen/context.py @@ -170,27 +170,29 @@ class Context: a parent context within which it can be created (the parent then becomes the first `__init__` method parameter). - A separate map of record variables is maintained because we will - generate named tuples for record types, and complex filtering of - e.g. nested maps of variables is awkward or impossible in Jinja. - TODO: make this a prerendering step - """ name: ContextName - base: Optional[type] - parent: Optional[Union[type, str]] - description: Optional[str] - metadata: List[Metadata] + definition: Dfn variables: Vars records: Vars - references: Refs + """ + A separate map of record variables is maintained because we will + generate named tuples for record types, and complex filtering of + e.g. nested maps of variables is awkward or impossible in Jinja. + TODO: make this a prerendering step + """ + base: Optional[type] = None + parent: Optional[Union[type, str]] = None + description: Optional[str] = None + reference: bool = False + references: Optional[Refs] = None def make_context( name: ContextName, - dfn: Dfn, - common: Optional[Dfn] = None, + definition: Dfn, + commonvars: Optional[Dfn] = None, references: Optional[Refs] = None, ) -> Context: """ @@ -210,11 +212,11 @@ def make_context( is related to. """ - common = common or dict() + commonvars = commonvars or dict() + reference = Ref.from_dfn(definition) references = references or dict() - ref = Ref.from_dfn(dfn) # this a ref? - refs = dict() # referenced contexts - records = dict() # record variables + referenced = dict() + records = dict() def _ntname(s): """ @@ -244,17 +246,17 @@ def _parent() -> Optional[str]: the need to import `MFSimulation` in this file (avoids potential for circular imports). """ - l, r = dfn.name + l, r = definition.name if (l, r) == ("sim", "nam") and name == ("sim", "nam"): return None if l in ["sim", "exg", "sln"]: return "MFSimulation" if name.r is None: return "MFSimulation" - if ref: - if len(ref.parents) > 1: - return f"Union[{', '.join([_try_get_type_name(t) for t in ref.parents])}]" - return ref.parents[0] + if reference: + if len(reference.parents) > 1: + return f"Union[{', '.join([_try_get_type_name(t) for t in reference.parents])}]" + return reference.parents[0] return "MFModel" parent = _parent() @@ -315,7 +317,7 @@ def _description(descr: str) -> str: if replace: key, _, subs = tail.strip().partition(" ") subs = literal_eval(subs) - cmn_var = common.get(key, None) + cmn_var = commonvars.get(key, None) if cmn_var is None: raise ValueError(f"Common variable not found: {key}") descr = cmn_var.get("description", "") @@ -328,11 +330,11 @@ def _description(descr: str) -> str: def _fields(record_name: str) -> Vars: """Recursively load/convert a record's fields.""" - record = dfn[record_name] + record = definition[record_name] field_names = record["type"].split()[1:] fields: Dict[str, Var] = { n: _convert(field, wrap=False) - for n, field in dfn.items() + for n, field in definition.items() if n in field_names } field_names = list(fields.keys()) @@ -389,7 +391,7 @@ def _fields(record_name: str) -> Vars: # fields directly inside the recarray (implicit). list # data for unions/keystrings necessarily comes nested. - is_explicit_record = len(names) == 1 and dfn[names[0]][ + is_explicit_record = len(names) == 1 and definition[names[0]][ "type" ].startswith("record") @@ -398,7 +400,7 @@ def _is_implicit_scalar_record(): # only scalar fields types = [ _try_get_type_name(v["type"]) - for n, v in dfn.items() + for n, v in definition.items() if n in names ] scalar_types = list(_SCALAR_TYPES.keys()) @@ -406,7 +408,7 @@ def _is_implicit_scalar_record(): if is_explicit_record: record_name = names[0] - record_spec = dfn[record_name] + record_spec = definition[record_name] record = _convert(record_spec, wrap=False) children = {_ntname(record_name).lower(): record} type_ = Iterable[record._type] @@ -438,7 +440,9 @@ def _is_implicit_scalar_record(): type_ = Iterable[record_type] else: # implicit complex record (i.e. some fields are records or unions) - fields = {n: _convert(dfn[n], wrap=False) for n in names} + fields = { + n: _convert(definition[n], wrap=False) for n in names + } first = list(fields.values())[0] single = len(fields) == 1 record_name = first.name if single else _name @@ -473,7 +477,7 @@ def _is_implicit_scalar_record(): # union (product), children are record choices elif _type.startswith("keystring"): names = _type.split()[1:] - children = {n: _convert(dfn[n], wrap=True) for n in names} + children = {n: _convert(definition[n], wrap=True) for n in names} type_ = Union[tuple([c._type for c in children.values()])] # record (sum) type, children are fields @@ -565,7 +569,7 @@ def _is_implicit_scalar_record(): ref_ = references.get(_name, None) if ref_: var_.reference = ref_ - refs[_name] = ref_ + referenced[_name] = ref_ # if the var is a record, make a named tuple for it if is_record: @@ -599,7 +603,7 @@ def _variables() -> Vars: Variables may be added, depending on the context type. """ - vars_ = dfn.copy() + vars_ = definition.copy() vars_ = { name: _convert(var, wrap=False) for name, var in vars_.items() @@ -614,53 +618,29 @@ def _variables() -> Vars: # avoid name/reserved keyword collisions) return {v.name: v for v in vars_.values()} - def _metadata() -> List[Metadata]: - """ - Get a list of the class' original definition attributes - as a partial, internal reproduction of the DFN contents. - - Notes - ----- - Currently, generated classes have a `.dfn` property that - reproduces the corresponding DFN sans a few attributes. - This represents the DFN in raw form, before adapting to - Python, consolidating nested types, etc. - """ - - def _fmt_var(var: Union[Var, List[Var]]) -> List[str]: - exclude = ["longname", "description"] - - def _fmt_name(k, v): - return v.replace("-", "_") if k == "name" else v - - return [ - " ".join([k, str(_fmt_name(k, v))]).strip() - for k, v in var.items() - if k not in exclude - ] - - meta = dfn.metadata or list() - return [["header"] + [m for m in meta]] + [ - _fmt_var(var) for var in dfn.omd.values(multi=True) - ] - return Context( name=name, + definition=definition, + variables=_variables(), + records=records, base=name.base, parent=parent, description=name.description, - metadata=_metadata(), - variables=_variables(), - records=records, - references=refs, + reference=reference, + references=referenced, ) def make_contexts( - dfn: Dfn, - common: Optional[Dfn] = None, - refs: Optional[Refs] = None, + definition: Dfn, + commonvars: Optional[Dfn] = None, + references: Optional[Refs] = None, ) -> Iterator[Context]: """Generate one or more input contexts from the given input definition.""" - for name in get_context_names(dfn.name): - yield make_context(name=name, dfn=dfn, common=common, references=refs) + for name in get_context_names(definition.name): + yield make_context( + name=name, + definition=definition, + commonvars=commonvars, + references=references, + ) diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index 3b4478ccc8..8641d0eb21 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -1,8 +1,9 @@ from collections import UserDict +from collections.abc import MutableMapping from dataclasses import dataclass from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Tuple -from boltons.dictutils import OMD +from boltons.dictutils import OrderedMultiDict class DfnName(NamedTuple): @@ -26,7 +27,7 @@ class DfnName(NamedTuple): @dataclass -class Dfn(UserDict): +class Dfn(MutableMapping): """ An MF6 input definition. @@ -39,81 +40,99 @@ class Dfn(UserDict): This class should not be modified after loading. """ - name: Optional[DfnName] + variables: OrderedMultiDict metadata: Optional[Metadata] + name: Optional[DfnName] def __init__( self, variables: Iterable[Tuple[str, Dict[str, Any]]], - name: Optional[DfnName] = None, metadata: Optional[Metadata] = None, + name: Optional[DfnName] = None, ): - self.omd = OMD(variables) - self.name = name + self.variables = OrderedMultiDict(variables) self.metadata = metadata - super().__init__(self.omd) + self.name = name + def __getitem__(self, key): + return self.variables.getlist(key) + + def __setitem__(self, key, val): + self.variables.__setitem__(key, val) + + def __delitem__(self, key): + self.variables.__delitem__(key) + + def __len__(self): + return len(self.variables.values(multi=True)) + + def __iter__(self): + return iter(self.variables) + + def __repr__(self): + return self.variables.__repr__() + + @classmethod + def load(cls, f, name: Optional[DfnName] = None) -> "Dfn": + """ + Load an input definition from a definition file. + """ + + meta = None + vars_ = list() + var = dict() + + for line in f: + # remove whitespace/etc from the line + line = line.strip() + + # record context name and flopy metadata + # attributes, skip all other comment lines + if line.startswith("#"): + _, sep, tail = line.partition("flopy") + if sep == "flopy": + if meta is None: + meta = list() + tail = tail.strip() + if "solution_package" in tail: + tail = tail.split() + tail.pop(1) + meta.append(tail) + continue + _, sep, tail = line.partition("package-type") + if sep == "package-type": + if meta is None: + meta = list + meta.append(f"{sep} {tail.strip()}") + continue + _, sep, tail = line.partition("solution_package") + continue -Dfns = Dict[str, Dfn] + # if we hit a newline and the parameter dict + # is nonempty, we've reached the end of its + # block of attributes + if not any(line): + if any(var): + n = var["name"] + vars_.append((n, var)) + var = dict() + continue + # split the attribute's key and value and + # store it in the parameter dictionary + key, _, value = line.partition(" ") + if key == "default_value": + key = "default" + if value in ["true", "false"]: + value = value == "true" + var[key] = value -def load_dfn(f, name: Optional[DfnName] = None) -> Dfn: - """ - Load an input definition from a definition file. - """ + # add the final parameter + if any(var): + n = var["name"] + vars_.append((n, var)) - meta = None - vars_ = list() - var = dict() - - for line in f: - # remove whitespace/etc from the line - line = line.strip() - - # record context name and flopy metadata - # attributes, skip all other comment lines - if line.startswith("#"): - _, sep, tail = line.partition("flopy") - if sep == "flopy": - if meta is None: - meta = list() - tail = tail.strip() - if "solution_package" in tail: - tail = tail.split() - tail.pop(1) - meta.append(tail) - continue - _, sep, tail = line.partition("package-type") - if sep == "package-type": - if meta is None: - meta = list - meta.append(f"{sep} {tail.strip()}") - continue - _, sep, tail = line.partition("solution_package") - continue - - # if we hit a newline and the parameter dict - # is nonempty, we've reached the end of its - # block of attributes - if not any(line): - if any(var): - n = var["name"] - vars_.append((n, var)) - var = dict() - continue - - # split the attribute's key and value and - # store it in the parameter dictionary - key, _, value = line.partition(" ") - if key == "default_value": - key = "default" - if value in ["true", "false"]: - value = value == "true" - var[key] = value - - # add the final parameter - if any(var): - n = var["name"] - vars_.append((n, var)) - - return Dfn(variables=vars_, name=name, metadata=meta) + return cls(variables=vars_, name=name, metadata=meta) + + +Dfns = Dict[str, Dfn] diff --git a/flopy/mf6/utils/codegen/make.py b/flopy/mf6/utils/codegen/make.py index 52a296a1ad..d9bcee70bb 100644 --- a/flopy/mf6/utils/codegen/make.py +++ b/flopy/mf6/utils/codegen/make.py @@ -13,7 +13,7 @@ make_context, make_contexts, ) -from flopy.mf6.utils.codegen.dfn import Dfn, DfnName, Dfns, load_dfn +from flopy.mf6.utils.codegen.dfn import Dfn, DfnName, Dfns from flopy.mf6.utils.codegen.ref import Ref, Refs _TEMPLATE_LOADER = PackageLoader("flopy", "mf6/utils/templates/") @@ -31,7 +31,9 @@ def make_targets( ): """Generate Python source file(s) from the given input definition.""" - for context in make_contexts(dfn=dfn, common=common, refs=refs): + for context in make_contexts( + definition=dfn, commonvars=common, references=refs + ): target = outdir / context.name.target with open(target, "w") as f: source = _TEMPLATE.render(**context.render()) @@ -55,7 +57,7 @@ def make_all(dfndir: Path, outdir: Path, verbose: bool = False): common = None else: with open(common_path, "r") as f: - common = load_dfn(f) + common = Dfn.load(f) # load all the input definitions before we generate input # contexts so we can create foreign key refs between them. @@ -64,7 +66,7 @@ def make_all(dfndir: Path, outdir: Path, verbose: bool = False): for p in paths: name = DfnName(*p.stem.split("-")) with open(p) as f: - dfn = load_dfn(f, name=name) + dfn = Dfn.load(f, name=name) dfns[name] = dfn ref = Ref.from_dfn(dfn) if ref: diff --git a/flopy/mf6/utils/codegen/render.py b/flopy/mf6/utils/codegen/render.py index 953a26bdc2..146abe925f 100644 --- a/flopy/mf6/utils/codegen/render.py +++ b/flopy/mf6/utils/codegen/render.py @@ -15,9 +15,9 @@ def renderable( maybe_cls=None, *, - add_entry: Optional[Iterable[Tuple[Predicate, Entries]]] = None, keep_none: Optional[Iterable[str]] = None, quote_str: Optional[Iterable[str]] = None, + set_pairs: Optional[Iterable[Tuple[Predicate, Entries]]] = None, transform: Optional[Iterable[Tuple[Predicate, Transform]]] = None, type_name: Optional[Iterable[str]] = None, ): @@ -110,9 +110,9 @@ def renderable( """ - add_entry = add_entry or list() quote_str = quote_str or list() keep_none = keep_none or list() + set_pairs = set_pairs or list() transform = transform or list() type_name = type_name or list() @@ -140,7 +140,7 @@ def _dict(o): if p(o): d = t(o) - for p, e in add_entry: + for p, e in set_pairs: if not p(d): continue if e is None: diff --git a/flopy/mf6/utils/codegen/shim.py b/flopy/mf6/utils/codegen/shim.py index 3c68c7e663..ed77e983d3 100644 --- a/flopy/mf6/utils/codegen/shim.py +++ b/flopy/mf6/utils/codegen/shim.py @@ -1,10 +1,11 @@ import os from os import PathLike -from typing import Iterable, Optional, Union, get_args, get_origin +from typing import Iterable, List, Optional, Union, get_args, get_origin from numpy.typing import ArrayLike from pandas import DataFrame +from flopy.mf6.utils.codegen.dfn import Metadata from flopy.mf6.utils.codegen.spec import Var, VarKind @@ -439,19 +440,45 @@ def _loose_type(o) -> type: return d["_type"] +def _dfn(o) -> List[Metadata]: + """ + Get a list of the class' original definition attributes + as a partial, internal reproduction of the DFN contents. + + Notes + ----- + Currently, generated classes have a `.dfn` property that + reproduces the corresponding DFN sans a few attributes. + This represents the DFN in raw form, before adapting to + Python, consolidating nested types, etc. + """ + + d = dict(o) + dfn = d["definition"] + + def _fmt_var(var: Union[Var, List[Var]]) -> List[str]: + exclude = ["longname", "description"] + + def _fmt_name(k, v): + return v.replace("-", "_") if k == "name" else v + + return [ + " ".join([k, str(_fmt_name(k, v))]).strip() + for k, v in var.items() + if k not in exclude + ] + + meta = dfn["metadata"] or list() + return [["header"] + [m for m in meta]] + [ + _fmt_var(var) for var in dfn["data"].values() + ] + + SHIM = { - "keep_none": ["default", "block"], + "keep_none": ["default", "block", "metadata"], "quote_str": ["default"], - "type_name": ["_type"], - "transform": [ - # context-specific parameters - # for the `__init__()` method. - # do it as a `transform` (not - # `add_entry`) so we are able - # to control the param order. - (_is_ctx, _add_ctx_vars) - ], - "add_entry": [ + "set_pairs": [ + (_is_ctx, [("dfn", _dfn)]), ( _is_var, [ @@ -465,6 +492,15 @@ def _loose_type(o) -> type: ], ), ], + "type_name": ["_type"], + "transform": [ + # context-specific parameters + # for the `__init__()` method. + # do it as a `transform` (not + # `set_pairs`) so we are able + # to control the param order. + (_is_ctx, _add_ctx_vars) + ], } """ Arguments for `renderable` as applied to `Context` diff --git a/flopy/mf6/utils/templates/attrs.jinja b/flopy/mf6/utils/templates/attrs.jinja index bca097339a..b05a8e19d7 100644 --- a/flopy/mf6/utils/templates/attrs.jinja +++ b/flopy/mf6/utils/templates/attrs.jinja @@ -13,5 +13,5 @@ package_abbr = "{% if name.l != "sln" and name.l != "sim" and name.l != "exg" and name.l is not none %}{{ name.l }}{% endif %}{{ name.r }}" _package_type = "{{ name.r }}" dfn_file_name = "{% if name.l is not none %}{{ name.l }}-{% elif name.l is none %}sim-{% endif %}{{ name.r }}.dfn" - dfn = {{ metadata|pprint|indent(10) }} + dfn = {{ dfn|pprint|indent(10) }} {% endif -%} \ No newline at end of file diff --git a/flopy/mf6/utils/templates/context.py.jinja b/flopy/mf6/utils/templates/context.py.jinja index b8718f7292..22a075ec84 100644 --- a/flopy/mf6/utils/templates/context.py.jinja +++ b/flopy/mf6/utils/templates/context.py.jinja @@ -24,6 +24,6 @@ class {% if base == "MFSimulationBase" %}MF{% else %}Modflow{% endif %}{{ name.t {% include "load.jinja" %} -{% if subpkg and name.r != "hpc" %} +{% if reference is not none and name.r != "hpc" %} {% include "packages.jinja" %} {% endif %} \ No newline at end of file