From da0614a9a94e00c30f54ff2bb656290b4ceced42 Mon Sep 17 00:00:00 2001 From: spaulins-usgs Date: Mon, 31 Aug 2020 14:30:54 -0700 Subject: [PATCH 1/4] fix(extraneous package data): extraneous package data is stored as comments --- flopy/mf6/data/mfdatautil.py | 12 ++-- flopy/mf6/mfpackage.py | 124 ++++++++++++++++++++++++++--------- 2 files changed, 100 insertions(+), 36 deletions(-) diff --git a/flopy/mf6/data/mfdatautil.py b/flopy/mf6/data/mfdatautil.py index dcc30cc01c..411cfbf076 100644 --- a/flopy/mf6/data/mfdatautil.py +++ b/flopy/mf6/data/mfdatautil.py @@ -297,10 +297,12 @@ def __init__(self, comment, path, sim_data, line_number=0): text to add """ - def add_text(self, additional_text): + def add_text(self, additional_text, new_line=False): if additional_text: if isinstance(self.text, list): self.text.append(additional_text) + elif new_line: + self.text = "{}{}".format(self.text, additional_text) else: self.text = "{} {}".format(self.text, additional_text) @@ -376,12 +378,12 @@ def write(self, fd, eoln_suffix=True): def is_empty(self, include_whitespace=True): if include_whitespace: if self.text(): - return True - return False + return False + return True else: if self.text.strip(): - return True - return False + return False + return True """ Check text to see if it is valid comment text diff --git a/flopy/mf6/mfpackage.py b/flopy/mf6/mfpackage.py index 1fcd00402f..236e9d7c0b 100644 --- a/flopy/mf6/mfpackage.py +++ b/flopy/mf6/mfpackage.py @@ -75,12 +75,31 @@ def __init__( if simulation_data is None: self.comment = comment self.simulation_data = None - self.path = None + self.path = path self.comment_path = None else: self.connect_to_dict(simulation_data, path, comment) # TODO: Get data_items from dictionary self.data_items = [] + # build block comment paths + self.blk_trailing_comment_path = ("blk_trailing_comment",) + self.blk_post_comment_path = ("blk_post_comment",) + if path is not None: + self.blk_trailing_comment_path = path + (name, + "blk_trailing_comment",) + self.blk_post_comment_path = path + (name, "blk_post_comment",) + if self.blk_trailing_comment_path not in simulation_data.mfdata: + simulation_data.mfdata[ + self.blk_trailing_comment_path] = MFComment( + "", "", simulation_data, 0 + ) + if self.blk_post_comment_path not in simulation_data.mfdata: + simulation_data.mfdata[self.blk_post_comment_path] = MFComment( + "\n", "", simulation_data, 0 + ) + else: + self.blk_trailing_comment_path = ("blk_trailing_comment",) + self.blk_post_comment_path = ("blk_post_comment",) def build_header_variables( self, @@ -119,7 +138,15 @@ def build_header_variables( dimensions, fixed_data, ) + + self.add_data_item(new_data, data) + + def add_data_item(self, new_data, data): self.data_items.append(new_data) + if not isinstance(data, tuple): + data = (data,) + self.blk_trailing_comment_path += data + self.blk_post_comment_path += data def is_same_header(self, block_header): if len(self.variable_strings) > 0: @@ -318,16 +345,6 @@ def __init__( self.path = path self.datasets = OrderedDict() self.datasets_keyword = {} - self.blk_trailing_comment_path = path + ("blk_trailing_comment",) - self.blk_post_comment_path = path + ("blk_post_comment",) - if self.blk_trailing_comment_path not in simulation_data.mfdata: - simulation_data.mfdata[self.blk_trailing_comment_path] = MFComment( - "", "", simulation_data, 0 - ) - if self.blk_post_comment_path not in simulation_data.mfdata: - simulation_data.mfdata[self.blk_post_comment_path] = MFComment( - "\n", "", simulation_data, 0 - ) # initially disable if optional self.enabled = structure.number_non_optional_data() > 0 self.loaded = False @@ -581,6 +598,7 @@ def _build_repeating_header(self, header_data): self.block_headers.append(block_header) else: block_header_path = self.path + (len(self.block_headers),) + struct = self.structure last_header = self.block_headers[-1] try: @@ -615,7 +633,10 @@ def _new_dataset( # stress periods are stored 0 based initial_val = int(initial_val[0]) - 1 if isinstance(initial_val, list): + initial_val_path = tuple(initial_val) initial_val = [tuple(initial_val)] + else: + initial_val_path = initial_val try: new_data = MFBlock.data_factory( self._simulation_data, @@ -636,7 +657,8 @@ def _new_dataset( ' dataset "{}" to block ' '"{}"'.format(dataset_struct.name, self.structure.name), ) - self.block_headers[-1].data_items.append(new_data) + self.block_headers[-1].add_data_item(new_data, initial_val_path) + else: try: self.datasets[key] = self.data_factory( @@ -747,6 +769,9 @@ def load(self, block_header, fd, strict=True): line = fd_block.readline() datautil.PyListUtil.reset_delimiter_used() arr_line = datautil.PyListUtil.split_data_line(line) + post_data_comments = MFComment( + "", "", self._simulation_data, 0 + ) while MFComment.is_comment(line, True): initial_comment.add_text(line) line = fd_block.readline() @@ -859,9 +884,6 @@ def load(self, block_header, fd, strict=True): else: arr_line = "" # capture any trailing comments - post_data_comments = MFComment( - "", "", self._simulation_data, 0 - ) dataset.post_data_comments = post_data_comments while arr_line and ( len(next_line[1]) <= 2 or arr_line[0][:3].upper() != "END" @@ -914,10 +936,6 @@ def load(self, block_header, fd, strict=True): and result[1][:3].upper() == "END" ): break - - self._simulation_data.mfdata[ - self.blk_trailing_comment_path - ].text = comments self.loaded = True self.is_valid() @@ -1114,6 +1132,7 @@ def write(self, fd, ext_file_action=ExtFileAction.copy_relative_paths): if len(repeating_datasets) > 0: # loop through all block headers for block_header in self.block_headers: + # write block self._write_block(fd, block_header, ext_file_action) else: # write out block @@ -1248,7 +1267,10 @@ def _write_block(self, fd, block_header, ext_file_action): ), ) # write trailing comments - self._simulation_data.mfdata[self.blk_trailing_comment_path].write(fd) + if block_header.blk_trailing_comment_path in \ + self._simulation_data.mfdata: + self._simulation_data.mfdata[ + block_header.blk_trailing_comment_path].write(fd) if self.external_file_name is not None: # switch back writing to package file @@ -1259,7 +1281,8 @@ def _write_block(self, fd, block_header, ext_file_action): block_header.write_footer(fd) # write post block comments - self._simulation_data.mfdata[self.blk_post_comment_path].write(fd) + self._simulation_data.mfdata[ + block_header.blk_post_comment_path].write(fd) # write extra line if comments are off if not self._simulation_data.comments_on: @@ -1699,7 +1722,8 @@ def _get_block_header_info(self, line, path): ) elif len(arr_clean_line) == 2: return MFBlockHeader( - arr_clean_line[1], header_variable_strs, header_comment + arr_clean_line[1], header_variable_strs, header_comment, + self._simulation_data, path ) else: # process text after block name @@ -1715,7 +1739,8 @@ def _get_block_header_info(self, line, path): else: header_variable_strs.append(entry) return MFBlockHeader( - arr_clean_line[1], header_variable_strs, header_comment + arr_clean_line[1], header_variable_strs, header_comment, + self._simulation_data, path ) def _update_size_defs(self): @@ -2049,11 +2074,9 @@ def _load_blocks(self, fd_input_file, strict=True, max_blocks=sys.maxsize): break else: found_first_block = True - self.post_block_comments = MFComment( - "", self.path, self._simulation_data - ) skip_block = False - if self.blocks[block_key].loaded: + cur_block = self.blocks[block_key] + if cur_block.loaded: # Only blocks defined as repeating are allowed to have # multiple entries header_name = block_header_info.name @@ -2074,6 +2097,17 @@ def _load_blocks(self, fd_input_file, strict=True, max_blocks=sys.maxsize): ) print(warning_str) skip_block = True + bhs = cur_block.structure.block_header_structure + bhval = block_header_info.variable_strings + if len(bhs) > 0 and len(bhval) > 0 and \ + bhs[0].name == 'iper': + nper = self._simulation_data.mfdata[ + ("tdis", "dimensions", "nper")].get_data() + bhval_int = datautil.DatumUtil.is_int(bhval[0]) + if not bhval_int or int(bhval[0]) > nper: + # skip block when block stress period is greater + # than nper + skip_block = True if not skip_block: if ( @@ -2082,20 +2116,48 @@ def _load_blocks(self, fd_input_file, strict=True, max_blocks=sys.maxsize): ): print( " loading block {}...".format( - self.blocks[block_key].structure.name + cur_block.structure.name ) ) + # reset comments + self.post_block_comments = MFComment( + "", self.path, self._simulation_data + ) - self.blocks[block_key].load( + cur_block.load( block_header_info, fd_input_file, strict ) + + # write post block comment comment self._simulation_data.mfdata[ - self.blocks[block_key].blk_post_comment_path - ] = self.post_block_comments + cur_block.block_headers[-1]. \ + blk_post_comment_path] \ + = self.post_block_comments blocks_read += 1 if blocks_read >= max_blocks: break + else: + # treat skipped block as if it is all comments + arr_line = datautil.PyListUtil.split_data_line( + clean_line) + self.post_block_comments.add_text( + "{}".format(line), True) + while arr_line and ( + len(line) <= 2 or + arr_line[0][:3].upper() != "END" + ): + line = fd_input_file.readline() + arr_line = datautil.PyListUtil.split_data_line( + line.strip() + ) + if arr_line: + self.post_block_comments\ + .add_text("{}".format(line), True) + self._simulation_data.mfdata[ + cur_block.block_headers[-1].blk_post_comment_path + ] = self.post_block_comments + else: if not ( len(clean_line) == 0 From 803f060e994fafe6678802059936e01ffce07bd8 Mon Sep 17 00:00:00 2001 From: spaulins-usgs Date: Thu, 3 Sep 2020 09:38:25 -0700 Subject: [PATCH 2/4] fix(loading list data): fix handles case where jagged data in a list has size zero --- flopy/mf6/data/mffileaccess.py | 18 ++++++-- flopy/mf6/mfpackage.py | 82 +++++++++++++++++++++------------- 2 files changed, 65 insertions(+), 35 deletions(-) diff --git a/flopy/mf6/data/mffileaccess.py b/flopy/mf6/data/mffileaccess.py index eb32076b8f..eb33e0fd70 100644 --- a/flopy/mf6/data/mffileaccess.py +++ b/flopy/mf6/data/mffileaccess.py @@ -1552,10 +1552,20 @@ def _load_list_line( # comment mark found and expecting optional # data_item, we are done break - if ( - data_index >= arr_line_len - and data_item.optional - ): + if data_index >= arr_line_len: + if data_item.optional: + break + else: + unknown_repeats = ( + storage.resolve_shape_list( + data_item, + repeat_count, + current_key, + data_line, + )[1] + ) + if unknown_repeats: + break break more_data_expected = True unknown_repeats = False diff --git a/flopy/mf6/mfpackage.py b/flopy/mf6/mfpackage.py index 236e9d7c0b..5b5352f0bd 100644 --- a/flopy/mf6/mfpackage.py +++ b/flopy/mf6/mfpackage.py @@ -84,15 +84,21 @@ def __init__( # build block comment paths self.blk_trailing_comment_path = ("blk_trailing_comment",) self.blk_post_comment_path = ("blk_post_comment",) + if isinstance(path, list): + path = tuple(path) if path is not None: - self.blk_trailing_comment_path = path + (name, - "blk_trailing_comment",) - self.blk_post_comment_path = path + (name, "blk_post_comment",) + self.blk_trailing_comment_path = path + ( + name, + "blk_trailing_comment", + ) + self.blk_post_comment_path = path + ( + name, + "blk_post_comment", + ) if self.blk_trailing_comment_path not in simulation_data.mfdata: simulation_data.mfdata[ - self.blk_trailing_comment_path] = MFComment( - "", "", simulation_data, 0 - ) + self.blk_trailing_comment_path + ] = MFComment("", "", simulation_data, 0) if self.blk_post_comment_path not in simulation_data.mfdata: simulation_data.mfdata[self.blk_post_comment_path] = MFComment( "\n", "", simulation_data, 0 @@ -143,6 +149,11 @@ def build_header_variables( def add_data_item(self, new_data, data): self.data_items.append(new_data) + while isinstance(data, list): + if len(data) > 0: + data = data[0] + else: + data = None if not isinstance(data, tuple): data = (data,) self.blk_trailing_comment_path += data @@ -769,9 +780,7 @@ def load(self, block_header, fd, strict=True): line = fd_block.readline() datautil.PyListUtil.reset_delimiter_used() arr_line = datautil.PyListUtil.split_data_line(line) - post_data_comments = MFComment( - "", "", self._simulation_data, 0 - ) + post_data_comments = MFComment("", "", self._simulation_data, 0) while MFComment.is_comment(line, True): initial_comment.add_text(line) line = fd_block.readline() @@ -1267,10 +1276,9 @@ def _write_block(self, fd, block_header, ext_file_action): ), ) # write trailing comments - if block_header.blk_trailing_comment_path in \ - self._simulation_data.mfdata: - self._simulation_data.mfdata[ - block_header.blk_trailing_comment_path].write(fd) + pth = block_header.blk_trailing_comment_path + if pth in self._simulation_data.mfdata: + self._simulation_data.mfdata[pth].write(fd) if self.external_file_name is not None: # switch back writing to package file @@ -1281,8 +1289,9 @@ def _write_block(self, fd, block_header, ext_file_action): block_header.write_footer(fd) # write post block comments - self._simulation_data.mfdata[ - block_header.blk_post_comment_path].write(fd) + pth = block_header.blk_post_comment_path + if pth in self._simulation_data.mfdata: + self._simulation_data.mfdata[pth].write(fd) # write extra line if comments are off if not self._simulation_data.comments_on: @@ -1722,8 +1731,11 @@ def _get_block_header_info(self, line, path): ) elif len(arr_clean_line) == 2: return MFBlockHeader( - arr_clean_line[1], header_variable_strs, header_comment, - self._simulation_data, path + arr_clean_line[1], + header_variable_strs, + header_comment, + self._simulation_data, + path, ) else: # process text after block name @@ -1739,8 +1751,11 @@ def _get_block_header_info(self, line, path): else: header_variable_strs.append(entry) return MFBlockHeader( - arr_clean_line[1], header_variable_strs, header_comment, - self._simulation_data, path + arr_clean_line[1], + header_variable_strs, + header_comment, + self._simulation_data, + path, ) def _update_size_defs(self): @@ -2099,10 +2114,14 @@ def _load_blocks(self, fd_input_file, strict=True, max_blocks=sys.maxsize): skip_block = True bhs = cur_block.structure.block_header_structure bhval = block_header_info.variable_strings - if len(bhs) > 0 and len(bhval) > 0 and \ - bhs[0].name == 'iper': + if ( + len(bhs) > 0 + and len(bhval) > 0 + and bhs[0].name == "iper" + ): nper = self._simulation_data.mfdata[ - ("tdis", "dimensions", "nper")].get_data() + ("tdis", "dimensions", "nper") + ].get_data() bhval_int = datautil.DatumUtil.is_int(bhval[0]) if not bhval_int or int(bhval[0]) > nper: # skip block when block stress period is greater @@ -2130,9 +2149,8 @@ def _load_blocks(self, fd_input_file, strict=True, max_blocks=sys.maxsize): # write post block comment comment self._simulation_data.mfdata[ - cur_block.block_headers[-1]. \ - blk_post_comment_path] \ - = self.post_block_comments + cur_block.block_headers[-1].blk_post_comment_path + ] = self.post_block_comments blocks_read += 1 if blocks_read >= max_blocks: @@ -2140,20 +2158,22 @@ def _load_blocks(self, fd_input_file, strict=True, max_blocks=sys.maxsize): else: # treat skipped block as if it is all comments arr_line = datautil.PyListUtil.split_data_line( - clean_line) + clean_line + ) self.post_block_comments.add_text( - "{}".format(line), True) + "{}".format(line), True + ) while arr_line and ( - len(line) <= 2 or - arr_line[0][:3].upper() != "END" + len(line) <= 2 or arr_line[0][:3].upper() != "END" ): line = fd_input_file.readline() arr_line = datautil.PyListUtil.split_data_line( line.strip() ) if arr_line: - self.post_block_comments\ - .add_text("{}".format(line), True) + self.post_block_comments.add_text( + "{}".format(line), True + ) self._simulation_data.mfdata[ cur_block.block_headers[-1].blk_post_comment_path ] = self.post_block_comments From 17adafe1d2319cd8ae490ef606b2ece58d4f9b00 Mon Sep 17 00:00:00 2001 From: spaulins-usgs Date: Thu, 3 Sep 2020 10:35:04 -0700 Subject: [PATCH 3/4] fix(reading repeating data): variable initialization fix --- flopy/mf6/data/mffileaccess.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flopy/mf6/data/mffileaccess.py b/flopy/mf6/data/mffileaccess.py index eb33e0fd70..0499d5cd7f 100644 --- a/flopy/mf6/data/mffileaccess.py +++ b/flopy/mf6/data/mffileaccess.py @@ -1502,6 +1502,7 @@ def _load_list_line( else: # read variables var_index = 0 + repeat_count = 0 data = "" for data_item_index, data_item in enumerate( data_set.data_item_structures From da570baa5732c40cde63ff8e2f733d71402a4c68 Mon Sep 17 00:00:00 2001 From: spaulins-usgs Date: Thu, 3 Sep 2020 12:11:48 -0700 Subject: [PATCH 4/4] fix(test update): update test to properly function with the way flopy now treats extra stress periods as comment text --- autotest/t504_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autotest/t504_test.py b/autotest/t504_test.py index 0d359a5c21..5f2448845a 100644 --- a/autotest/t504_test.py +++ b/autotest/t504_test.py @@ -255,7 +255,7 @@ def test005_advgw_tidal(): # add a stress period beyond nper spd = ghb.stress_period_data.get_data() - spd[20] = copy.deepcopy(spd[9]) + spd[20] = copy.deepcopy(spd[0]) ghb.stress_period_data.set_data(spd) # make temp folder to save simulation