From cdd7ebd371cdf974be67f17857d769c8b6db9e30 Mon Sep 17 00:00:00 2001 From: mjreno Date: Tue, 19 Jul 2022 16:03:19 -0400 Subject: [PATCH 1/6] setup: add download, depencies and related scripts --- .gitignore | 4 + modflow_devtools/__init__.py | 27 + modflow_devtools/utilities/build_exes.py | 156 ++++ modflow_devtools/utilities/download.py | 965 ++++++++++++++++++++ modflow_devtools/utilities/get_exes.py | 117 +++ modflow_devtools/utilities/usgsprograms.py | 533 +++++++++++ modflow_devtools/utilities/usgsprograms.txt | 25 + 7 files changed, 1827 insertions(+) create mode 100644 modflow_devtools/utilities/build_exes.py create mode 100644 modflow_devtools/utilities/download.py create mode 100644 modflow_devtools/utilities/get_exes.py create mode 100644 modflow_devtools/utilities/usgsprograms.py create mode 100644 modflow_devtools/utilities/usgsprograms.txt diff --git a/.gitignore b/.gitignore index ad37b1a7..7bbfb94c 100644 --- a/.gitignore +++ b/.gitignore @@ -131,3 +131,7 @@ dmypy.json # pycharme .idea/ +# downloaded exe +modflow_devtools/bin +modflow_devtools/utilities/temp + diff --git a/modflow_devtools/__init__.py b/modflow_devtools/__init__.py index f666d663..ce7c227b 100644 --- a/modflow_devtools/__init__.py +++ b/modflow_devtools/__init__.py @@ -12,6 +12,10 @@ __status__, __version__, ) +from .common_regression import get_example_basedir, get_example_dirs, \ + get_home_dir, get_select_dirs, \ + get_select_packages, is_directory_available, \ + set_mf6_regression from .framework import running_on_CI, set_teardown_test, testing_framework from .simulation import Simulation from .targets import get_mf6_version, get_target_dictionary, run_exe @@ -47,10 +51,27 @@ write_head, ) from .utilities.disu_util import get_disu_kwargs +from .utilities.usgsprograms import usgs_program_data +from .utilities.download import ( + download_and_unzip, + get_repo_assets, + getmfexes, + getmfnightly, + repo_latest_version, + zip_all, +) # define public interfaces __all__ = [ "__version__", + # common_regression + "get_example_basedir", + "get_example_dirs", + "get_home_dir", + "get_select_dirs", + "get_select_packages", + "is_directory_available", + "set_mf6_regression", # targets "run_exe", "get_mf6_version", @@ -89,4 +110,10 @@ "write_head", "write_budget", "get_disu_kwargs", + "usgs_program_data", + "download_and_unzip", + "getmfexes", + "repo_latest_version", + "get_repo_assets", + "zip_all", ] diff --git a/modflow_devtools/utilities/build_exes.py b/modflow_devtools/utilities/build_exes.py new file mode 100644 index 00000000..7333858c --- /dev/null +++ b/modflow_devtools/utilities/build_exes.py @@ -0,0 +1,156 @@ +# Build targets + +# to use ifort on windows, run this +# python build_exes.py -fc ifort + +# can compile only mf6 directly using this command: +# python -c "import build_exes; build_exes.test_build_modflow6()" + +import os +import pathlib as pl +import subprocess as sp +import sys +from contextlib import contextmanager + +from modflow_devtools import running_on_CI + +if running_on_CI(): + print("running on CI environment") + os.environ["PYMAKE_DOUBLE"] = "1" + +# set OS dependent extensions +eext = "" +soext = ".so" +if sys.platform.lower() == "win32": + eext = ".exe" + soext = ".dll" +elif sys.platform.lower() == "darwin": + soext = ".dylib" + +mfexe_pth = "temp/mfexes" + +# use the line below to set fortran compiler using environmental variables +# os.environ["FC"] = "ifort" + +# some flags to check for errors in the code +# add -Werror for compilation to terminate if errors are found +strict_flags = ( + "-fall-intrinsics " + "-Wtabs -Wline-truncation -Wunused-label " + "-Wunused-variable -pedantic -std=f2008 " + "-Wcharacter-truncation" +) + + +@contextmanager +def set_directory(path: str): + """Sets the cwd within the context + + Args: + path (Path): The path to the cwd + + Yields: + None + """ + + origin = os.path.abspath(os.getcwd()) + path = os.path.abspath(path) + try: + os.chdir(path) + print(f"change from {origin} -> {path}") + yield + finally: + os.chdir(origin) + print(f"change from {path} -> {origin}") + + +def relpath_fallback(pth): + try: + # throws ValueError on Windows if pth is on a different drive + return os.path.relpath(pth) + except ValueError: + return os.path.abspath(pth) + + +def create_dir(pth): + # create pth directory + print(f"creating... {os.path.abspath(pth)}") + os.makedirs(pth, exist_ok=True) + + msg = f"could not create... {os.path.abspath(pth)}" + assert os.path.exists(pth), msg + + +def set_compiler_environment_variable(): + fc = None + + # parse command line arguments + for idx, arg in enumerate(sys.argv): + if arg.lower() == "-fc": + fc = sys.argv[idx + 1] + elif arg.lower().startswith("-fc="): + fc = arg.split("=")[1] + + # determine if fc needs to be set to the FC environmental variable + env_var = os.getenv("FC", default="gfortran") + if fc is None and fc != env_var: + fc = env_var + + # validate Fortran compiler + fc_options = ( + "gfortran", + "ifort", + ) + if fc not in fc_options: + raise ValueError( + f"Fortran compiler {fc} not supported. Fortran compile must be " + + f"[{', '.join(str(value) for value in fc_options)}]." + ) + + # set FC environment variable + os.environ["FC"] = fc + + +def meson_build( + dir_path: str = "..", + libdir: str = "bin", +): + set_compiler_environment_variable() + is_windows = sys.platform.lower() == "win32" + with set_directory(dir_path): + cmd = ( + "meson setup builddir " + + f"--bindir={os.path.abspath(libdir)} " + + f"--libdir={os.path.abspath(libdir)} " + + "--prefix=" + ) + if is_windows: + cmd += "%CD%" + else: + cmd += "$(pwd)" + if pl.Path("builddir").is_dir(): + cmd += " --wipe" + print(f"setup meson\nrunning...\n {cmd}") + sp.run(cmd, shell=True, check=True) + + cmd = "meson install -C builddir" + print(f"build and install with meson\nrunning...\n {cmd}") + sp.run(cmd, shell=True, check=True) + + +def test_create_dirs(): + pths = [os.path.join("..", "bin"), os.path.join("temp")] + + for pth in pths: + create_dir(pth) + + return + + +def test_meson_build(): + meson_build() + + +if __name__ == "__main__": + test_create_dirs() + test_meson_build() diff --git a/modflow_devtools/utilities/download.py b/modflow_devtools/utilities/download.py new file mode 100644 index 00000000..d613d1f6 --- /dev/null +++ b/modflow_devtools/utilities/download.py @@ -0,0 +1,965 @@ +"""Utility functions to: + +1. download and unzip software releases from the USGS and other organizations + (triangle, MT3DMS). +2. download the latest MODFLOW-based applications and utilities for MacOS, + Linux, and Windows from https://github.com/MODFLOW-USGS/executables +3. determine the latest version (GitHub tag) of a GitHub repository and a + dictionary containing the file name and the link to a asset on + contained in a github repository +4. compress all files in a list, files in a list of directories + +""" +import os +import shutil +import sys +import tarfile +import time +import timeit +from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo + +import requests + + +class MFZipFile(ZipFile): + """ZipFile file attributes are not being preserved. This class preserves + file attributes as described on StackOverflow at + https://stackoverflow.com/questions/39296101/python-zipfile-removes-execute-permissions-from-binaries + + """ + + def extract(self, member, path=None, pwd=None): + """ + + Parameters + ---------- + member : str + individual file to extract. If member does not exist, all files + are extracted. + path : str + directory path to extract file in a zip file (default is None, + which results in files being extracted in the current directory) + pwd : str + zip file password (default is None) + + Returns + ------- + ret_val : int + return value indicating status of file extraction + + """ + if not isinstance(member, ZipInfo): + member = self.getinfo(member) + + if path is None: + path = os.getcwd() + + ret_val = self._extract_member(member, path, pwd) + attr = member.external_attr >> 16 + if attr != 0: + os.chmod(ret_val, attr) + + return ret_val + + def extractall(self, path=None, members=None, pwd=None): + """Extract all files in the zipfile. + + Parameters + ---------- + path : str + directory path to extract files in a zip file (default is None, + which results in files being extracted in the current directory) + members : str + individual files to extract (default is None, which extracts + all members) + pwd : str + zip file password (default is None) + + Returns + ------- + + """ + if members is None: + members = self.namelist() + + if path is None: + path = os.getcwd() + else: + if hasattr(os, "fspath"): + # introduced in python 3.6 and above + path = os.fspath(path) + + for zipinfo in members: + self.extract(zipinfo, path, pwd) + + @staticmethod + def compressall(path, file_pths=None, dir_pths=None, patterns=None): + """Compress selected files or files in selected directories. + + Parameters + ---------- + path : str + output zip file path + file_pths : str or list of str + file paths to include in the output zip file (default is None) + dir_pths : str or list of str + directory paths to include in the output zip file (default is None) + patterns : str or list of str + file patterns to include in the output zip file (default is None) + + Returns + ------- + success : bool + boolean indicating if the output zip file was created + + """ + + # create an empty list + if file_pths is None: + file_pths = [] + # convert files to a list + else: + if isinstance(file_pths, str): + file_pths = [file_pths] + elif isinstance(file_pths, tuple): + file_pths = list(file_pths) + + # remove directories from the file list + if len(file_pths) > 0: + file_pths = [e for e in file_pths if os.path.isfile(e)] + + # convert dirs to a list if a str (a tuple is allowed) + if dir_pths is None: + dir_pths = [] + else: + if isinstance(dir_pths, str): + dir_pths = [dir_pths] + + # convert find to a list if a str (a tuple is allowed) + if patterns is not None: + if isinstance(patterns, str): + patterns = [patterns] + + # walk through dirs and add files to the list + for dir_pth in dir_pths: + for dirname, subdirs, files in os.walk(dir_pth): + for filename in files: + fpth = os.path.join(dirname, filename) + # add the file if it does not exist in file_pths + if fpth not in file_pths: + file_pths.append(fpth) + + # remove file_paths that do not match the patterns + if patterns is not None: + tlist = [] + for file_pth in file_pths: + if any(p in os.path.basename(file_pth) for p in patterns): + tlist.append(file_pth) + file_pths = tlist + + # write the zipfile + success = True + if len(file_pths) > 0: + zf = ZipFile(path, "w", ZIP_DEFLATED) + + # write files to zip file + for file_pth in file_pths: + arcname = os.path.basename(file_pth) + zf.write(file_pth, arcname=arcname) + + # close the zip file + zf.close() + else: + msg = "No files to add to the zip file" + print(msg) + success = False + + return success + + +def _request_get(url, verify=True, timeout=1, max_requests=10, verbose=False): + """Make a url request + + Parameters + ---------- + url : str + url address for the zip file + verify : bool + boolean indicating if the url request should be verified + (default is True) + timeout : int + url request time out length (default is 1 seconds) + max_requests : int + number of url download request attempts (default is 10) + verbose : bool + boolean indicating if output will be printed to the terminal + (default is False) + + Returns + ------- + req : request object + request object for url + + """ + for idx in range(max_requests): + if verbose: + msg = f"open request attempt {idx + 1} of {max_requests}" + print(msg) + try: + req = requests.get( + url, stream=True, verify=verify, timeout=timeout + ) + except: + if idx < max_requests - 1: + time.sleep(13) + continue + else: + msg = "Cannot open request from:\n" + f" {url}\n\n" + print(msg) + req.raise_for_status() + + # successful request + break + + return req + + +def _request_header(url, max_requests=10, verbose=False): + """Get the headers from a url + + Parameters + ---------- + url : str + url address for the zip file + max_requests : int + number of url download request attempts (default is 10) + verbose : bool + boolean indicating if output will be printed to the terminal + (default is False) + + Returns + ------- + header : request header object + request header object for url + + """ + for idx in range(max_requests): + if verbose: + msg = f"open request attempt {idx + 1} of {max_requests}" + print(msg) + + header = requests.head(url, allow_redirects=True) + if header.status_code != 200: + if idx < max_requests - 1: + time.sleep(13) + continue + else: + msg = "Cannot open request from:\n" + f" {url}\n\n" + print(msg) + header.raise_for_status() + + # successful header request + break + + return header + + +def download_and_unzip( + url, + pth="./", + delete_zip=True, + verify=True, + timeout=30, + max_requests=10, + chunk_size=2048000, + verbose=False, +): + """Download and unzip a zip file from a url. + + Parameters + ---------- + url : str + url address for the zip file + pth : str + path where the zip file will be saved (default is the current path) + delete_zip : bool + boolean indicating if the zip file should be deleted after it is + unzipped (default is True) + verify : bool + boolean indicating if the url request should be verified + timeout : int + url request time out length (default is 30 seconds) + max_requests : int + number of url download request attempts (default is 10) + chunk_size : int + maximum url download request chunk size (default is 2048000 bytes) + verbose : bool + boolean indicating if output will be printed to the terminal + + Returns + ------- + + """ + + # create download directory + if not os.path.exists(pth): + if verbose: + print(f"Creating the directory:\n {pth}") + os.makedirs(pth) + + if verbose: + print(f"Attempting to download the file:\n {url}") + + # define the filename + file_name = os.path.join(pth, url.split("/")[-1]) + + # download the file + success = False + tic = timeit.default_timer() + + # open request + req = _request_get( + url, + verify=verify, + timeout=timeout, + max_requests=max_requests, + verbose=verbose, + ) + + # get content length, if available + tag = "Content-length" + if tag in req.headers: + file_size = req.headers[tag] + len_file_size = len(file_size) + file_size = int(file_size) + + bfmt = "{:" + f"{len_file_size}" + ",d}" + sbfmt = "{:>" + f"{len(bfmt.format(int(file_size)))}" + "s} bytes" + msg = f" file size: {sbfmt.format(bfmt.format(int(file_size)))}" + if verbose: + print(msg) + else: + file_size = 0.0 + + # download data from url + for idx in range(max_requests): + # print download attempt message + if verbose: + print(f" download attempt: {idx + 1}") + + # connection established - download the file + download_size = 0 + try: + with open(file_name, "wb") as f: + for chunk in req.iter_content(chunk_size=chunk_size): + if chunk: + # increment the counter + download_size += len(chunk) + + # write the chunk + f.write(chunk) + + # write information to the screen + if verbose: + if file_size > 0: + download_percent = float( + download_size + ) / float(file_size) + msg = ( + " downloaded " + + sbfmt.format(bfmt.format(download_size)) + + " of " + + bfmt.format(int(file_size)) + + " bytes" + + f" ({download_percent:10.4%})" + ) + else: + msg = ( + " downloaded " + + sbfmt.format(bfmt.format(download_size)) + + " bytes" + ) + print(msg) + else: + sys.stdout.write(".") + sys.stdout.flush() + + success = True + except: + # reestablish request + req = _request_get( + url, + verify=verify, + timeout=timeout, + max_requests=max_requests, + verbose=verbose, + ) + + # try to download the data again + continue + + # terminate the download attempt loop + if success: + break + + # write the total download time + toc = timeit.default_timer() + tsec = toc - tic + if verbose: + print(f"\ntotal download time: {tsec} seconds") + + if success: + if file_size > 0: + if verbose: + print(f"download speed: {file_size / (1e6 * tsec)} MB/s") + else: + msg = f"could not download...{url}" + raise ConnectionError(msg) + + # Unzip the file, and delete zip file if successful. + if "zip" in os.path.basename(file_name) or "exe" in os.path.basename( + file_name + ): + z = MFZipFile(file_name) + try: + # write a message + if not verbose: + sys.stdout.write("\n") + print(f"uncompressing...'{file_name}'") + + # extract the files + z.extractall(pth) + except: + p = "Could not unzip the file. Stopping." + raise Exception(p) + z.close() + elif "tar" in os.path.basename(file_name): + ar = tarfile.open(file_name) + ar.extractall(path=pth) + ar.close() + + # delete the zipfile + if delete_zip: + if verbose: + print("Deleting the zipfile...") + os.remove(file_name) + + if verbose: + print("Done downloading and extracting...\n") + + return success + + +def zip_all(path, file_pths=None, dir_pths=None, patterns=None): + """Compress all files in the user-provided list of file paths and directory + paths that match the provided file patterns. + + Parameters + ---------- + path : str + path of the zip file that will be created + file_pths : str or list + file path or list of file paths to be compressed + dir_pths : str or list + directory path or list of directory paths to search for files that + will be compressed + patterns : str or list + file pattern or list of file patterns s to match to when creating a + list of files that will be compressed + + Returns + ------- + + """ + return MFZipFile.compressall( + path, file_pths=file_pths, dir_pths=dir_pths, patterns=patterns + ) + + +def _get_zipname(platform): + """Determine zipfile name for platform. + + Parameters + ---------- + platform : str + Platform that will run the executables. Valid values include mac, + linux, win32 and win64. If platform is None, then routine will + download the latest asset from the github repository. + + Returns + ------- + zipfile : str + Name of zipfile for platform + + """ + if platform is None: + if sys.platform.lower() == "darwin": + platform = "mac" + elif sys.platform.lower().startswith("linux"): + platform = "linux" + elif "win" in sys.platform.lower(): + is_64bits = sys.maxsize > 2**32 + if is_64bits: + platform = "win64" + else: + platform = "win32" + else: + errmsg = ( + f"Could not determine platform. sys.platform is {sys.platform}" + ) + raise Exception(errmsg) + else: + msg = f"unknown platform detected ({platform})" + success = platform in ["mac", "linux", "win32", "win64"] + if not success: + raise ValueError(msg) + return f"{platform}.zip" + + +def _get_default_repo(): + """Return the default repo name. + + Returns + ------- + default_repo : str + default github repository repo name + + """ + return "MODFLOW-USGS/executables" + + +def _get_default_url(): + """Return the default executables url path. + + Returns + ------- + default_url : str + default url for executables repository repo name + + """ + + return ( + f"https://github.com/{_get_default_repo()}/" + + "releases/latest/download/" + ) + + +def _get_default_json(tag_name=None): + """Return a default github api json for the provided release tag_name in a + github repository. + + Parameters + ---------- + tag_name : str + github repository release tag + + Returns + ------- + json_obj : dict + json object (dictionary) with a tag_name and assets including + file names and download links + + """ + # initialize json_obj dictionary + json_obj = {"tag_name": tag_name} + + # create appropriate url + if tag_name is not None: + url = ( + f"https://github.com/{_get_default_repo()}/" + + f"releases/latest/download/{tag_name}/" + ) + else: + url = ( + f"https://github.com/{_get_default_repo()}/" + + "releases/latest/download/" + ) + + # define asset names and paths for assets + names = ["mac.zip", "linux.zip", "win32.zip", "win64.zip"] + paths = [url + p for p in names] + + assets_list = [] + for name, path in zip(names, paths): + assets_list.append({"name": name, "browser_download_url": path}) + json_obj["assets"] = assets_list + + return json_obj + + +def _get_request_json(request_url, verbose=False, verify=True): + """Process a url request and return a json if successful. + + Parameters + ---------- + request_url : str + url for request + verbose : bool + boolean indicating if output will be printed to the terminal + default is false + verify : bool + boolean indicating if the url request should be verified + + Returns + ------- + success : bool + boolean indicating if the requat failed + status_code: integer + request status code + json_obj : dict + json object + + """ + import json + + max_requests = 10 + json_obj = None + success = True + + # open request + req = _request_get( + request_url, max_requests=max_requests, verbose=verbose, verify=verify + ) + + # connection established - retrieve the json + if req.ok: + json_obj = json.loads(req.text or req.content) + else: + success = req.status_code == requests.codes.ok + + return success, req, json_obj + + +def _repo_json( + github_repo, tag_name=None, error_return=False, verbose=False, verify=True +): + """Return the github api json for the latest github release in a github + repository. + + Parameters + ---------- + github_repo : str + Repository name, such as MODFLOW-USGS/modflow6 + tag_name : str + github repository release tag + error_return : bool + boolean indicating if None will be returned if there are GitHub API + issues + verbose : bool + boolean indicating if output will be printed to the terminal + verify : bool + boolean indicating if the url request should be verified + + Returns + ------- + json_obj : dict + json object (dictionary) with a tag_name and assets including + file names and download links + + """ + repo_url = f"https://api.github.com/repos/{github_repo}" + + if tag_name is None: + request_url = f"{repo_url}/releases/latest" + else: + request_url = f"{repo_url}/releases" + success, _, json_cat = _get_request_json( + request_url, verbose=verbose, verify=verify + ) + if success: + request_url = None + for release in json_cat: + if release["tag_name"] == tag_name: + request_url = release["url"] + break + if request_url is None: + msg = ( + f"Could not find tag_name ('{tag_name}') " + + "in release catalog" + ) + if error_return: + print(msg) + return None + else: + raise Exception(msg) + else: + msg = "Could not get release catalog from " + request_url + if error_return: + if verbose: + print(msg) + return None + else: + raise Exception(msg) + + msg = "Requesting asset data " + if tag_name is not None: + msg += f"for tag_name '{tag_name}' " + msg += f"from: {request_url}" + if verbose: + print(msg) + + # process the request + success, req, json_obj = _get_request_json( + request_url, verbose=verbose, verify=verify + ) + + # evaluate request errors + if not success: + if github_repo == _get_default_repo(): + msg = f"will use default values for {github_repo}" + if verbose: + print(msg) + json_obj = _get_default_json(tag_name) + else: + msg = "Could not find json from " + request_url + if verbose: + print(msg) + if error_return: + json_obj = None + else: + req.raise_for_status() + + # return json object + return json_obj + + +def get_repo_assets( + github_repo=None, version=None, error_return=False, verify=True +): + """Return a dictionary containing the file name and the link to the asset + contained in a github repository. + + Parameters + ---------- + github_repo : str + Repository name, such as MODFLOW-USGS/modflow6. If github_repo is + None set to 'MODFLOW-USGS/executables' + version : str + github repository release tag + error_return : bool + boolean indicating if None will be returned if there are GitHub API + issues + verify : bool + boolean indicating if the url request should be verified + + Returns + ------- + result_dict : dict + dictionary of file names and links + + """ + if github_repo is None: + github_repo = _get_default_repo() + + # get json and extract assets + json_obj = _repo_json( + github_repo, tag_name=version, error_return=error_return, verify=verify + ) + if json_obj is None: + result_dict = None + else: + assets = json_obj["assets"] + + # build simple assets dictionary + result_dict = {} + for asset in assets: + k = asset["name"] + if version is None: + value = github_repo + f"/{k}" + else: + value = asset["browser_download_url"] + result_dict[k] = value + + return result_dict + + +def repo_latest_version(github_repo=None, verify=True): + """Return a string of the latest version number (tag) contained in a github + repository release. + + Parameters + ---------- + github_repo : str + Repository name, such as MODFLOW-USGS/modflow6. If github_repo is + None set to 'MODFLOW-USGS/executables' + + Returns + ------- + version : str + string with the latest version/tag number + + """ + if github_repo is None: + github_repo = _get_default_repo() + + # get json + json_obj = _repo_json(github_repo, verify=verify) + + return json_obj["tag_name"] + + +def getmfexes( + pth=".", + version=None, + platform=None, + exes=None, + verbose=False, + verify=True, +): + """Get the latest MODFLOW binary executables from a github site + (https://github.com/MODFLOW-USGS/executables) for the specified operating + system and put them in the specified path. + + Parameters + ---------- + pth : str + Location to put the executables (default is current working directory) + version : str + Version of the MODFLOW-USGS/executables release to use. If version is + None the github repo will be queried for the version number. + platform : str + Platform that will run the executables. Valid values include mac, + linux, win32 and win64. If platform is None, then routine will + download the latest asset from the github repository. + exes : str or list of strings + executable or list of executables to retain + verbose : bool + boolean indicating if output will be printed to the terminal + verify : bool + boolean indicating if the url request should be verified + + """ + # set download directory to path in case a selection of files + download_dir = pth + + # Determine the platform in order to construct the zip file name + zipname = _get_zipname(platform) + + # Evaluate exes keyword + if exes is not None: + download_dir = os.path.join(".", "download_dir") + if isinstance(exes, str): + exes = tuple(exes) + elif isinstance(exes, (int, float)): + msg = "exes keyword must be a string or a list/tuple of strings" + raise TypeError(msg) + + # Determine path for file download and then download and unzip + if version is None: + download_url = _get_default_url() + zipname + else: + assets = get_repo_assets( + github_repo=_get_default_repo(), version=version, verify=verify + ) + download_url = assets[zipname] + download_and_unzip( + download_url, + download_dir, + verbose=verbose, + verify=verify, + ) + + if exes is not None: + # make sure pth exists + if not os.path.exists(pth): + if verbose: + print(f"Creating the directory:\n {pth}") + os.makedirs(pth) + + # move select files to pth + for f in os.listdir(download_dir): + src = os.path.join(download_dir, f) + dst = os.path.join(pth, f) + for exe in exes: + if exe in f: + shutil.move(src, dst) + break + + # remove the download directory + if os.path.isdir(download_dir): + if verbose: + print("Removing folder " + download_dir) + shutil.rmtree(download_dir) + + return + + +def getmfnightly( + pth=".", + platform=None, + exes=None, + verbose=False, + verify=True, +): + """Get the latest MODFLOW 6 binary nightly-build executables from github + (https://github.com/MODFLOW-USGS/modflow6-nightly-build/) for the specified + operating system and put them in the specified path. + + Parameters + ---------- + pth : str + Location to put the executables (default is current working directory) + platform : str + Platform that will run the executables. Valid values include mac, + linux, win32 and win64. If platform is None, then routine will + download the latest asset from the github repository. + exes : str or list of strings + executable or list of executables to retain + verbose : bool + boolean indicating if output will be printed to the terminal + verify : bool + boolean indicating if the url request should be verified + + """ + # set download directory to path in case a selection of files + download_dir = pth + + # Determine the platform in order to construct the zip file name + zipname = _get_zipname(platform) + + # Evaluate exes keyword + if exes is not None: + download_dir = os.path.join(".", "download_dir") + if isinstance(exes, str): + exes = tuple(exes) + elif isinstance(exes, (int, float)): + msg = "exes keyword must be a string or a list/tuple of strings" + raise TypeError(msg) + + # Determine path for file download and then download and unzip + # https://github.com/MODFLOW-USGS/modflow6-nightly-build/releases/latest/download/ + download_url = ( + "https://github.com/MODFLOW-USGS/" + + "modflow6-nightly-build/releases/latest/download/" + + zipname + ) + download_and_unzip( + download_url, + download_dir, + verbose=verbose, + verify=verify, + ) + + if exes is not None: + # make sure pth exists + if not os.path.exists(pth): + if verbose: + print(f"Creating the directory:\n {pth}") + os.makedirs(pth) + + # move select files to pth + for f in os.listdir(download_dir): + src = os.path.join(download_dir, f) + dst = os.path.join(pth, f) + for exe in exes: + if exe in f: + shutil.move(src, dst) + break + + # remove the download directory + if os.path.isdir(download_dir): + if verbose: + print("Removing folder " + download_dir) + shutil.rmtree(download_dir) + + return diff --git a/modflow_devtools/utilities/get_exes.py b/modflow_devtools/utilities/get_exes.py new file mode 100644 index 00000000..7dfb8501 --- /dev/null +++ b/modflow_devtools/utilities/get_exes.py @@ -0,0 +1,117 @@ +# Get executables + +import os +import shutil + +from download import download_and_unzip, getmfexes +from usgsprograms import usgs_program_data + +from build_exes import meson_build +from modflow_devtools import running_on_CI + +if running_on_CI(): + print("running on CI environment") + os.environ["PYMAKE_DOUBLE"] = "1" + +# path to rebuilt executables for previous versions of MODFLOW +rebuilt_bindir = os.path.join("..", "bin", "rebuilt") + +if not os.path.exists(rebuilt_bindir): + os.makedirs(rebuilt_bindir) + +# paths to downloaded for previous versions of MODFLOW +downloaded_bindir = os.path.join("..", "bin", "downloaded") + +if not os.path.exists(downloaded_bindir): + os.makedirs(downloaded_bindir) + + +mfexe_pth = "temp/mfexes" + +# use the line below to set fortran compiler using environmental variables +# os.environ["FC"] = "gfortran" + +# some flags to check for errors in the code +# add -Werror for compilation to terminate if errors are found +strict_flags = ( + "-Wtabs -Wline-truncation -Wunused-label " + "-Wunused-variable -pedantic -std=f2008 " + "-Wcharacter-truncation" +) + + +def get_compiler_envvar(fc): + env_var = os.environ.get("FC") + if env_var is not None: + if env_var != fc: + fc = env_var + return fc + + +def create_dir(pth): + # create pth directory + print(f"creating... {os.path.abspath(pth)}") + os.makedirs(pth, exist_ok=True) + + msg = f"could not create... {os.path.abspath(pth)}" + assert os.path.exists(pth), msg + + +def rebuild_mf6_release(): + target = "mf6" + download_pth = os.path.join("temp") + target_dict = usgs_program_data.get_target(target) + + download_and_unzip( + target_dict["url"], + pth=download_pth, + verbose=True, + ) + + # update IDEVELOP MODE in the release + srcpth = os.path.join( + download_pth, target_dict["dirname"], target_dict["srcdir"] + ) + fpth = os.path.join(srcpth, "Utilities", "version.f90") + with open(fpth) as f: + lines = f.read().splitlines() + assert len(lines) > 0, f"could not update {srcpth}" + + f = open(fpth, "w") + for line in lines: + tag = "IDEVELOPMODE = 0" + if tag in line: + line = line.replace(tag, "IDEVELOPMODE = 1") + f.write(f"{line}\n") + f.close() + + # build release source files with Meson + root_path = os.path.join(download_pth, target_dict["dirname"]) + meson_build(dir_path=root_path, libdir=os.path.abspath(rebuilt_bindir)) + + +def test_create_dirs(): + pths = [os.path.join("..", "bin"), os.path.join("temp")] + + for pth in pths: + create_dir(pth) + + +def test_getmfexes(verify=True): + getmfexes(mfexe_pth, verify=verify) + for target in os.listdir(mfexe_pth): + srcpth = os.path.join(mfexe_pth, target) + if os.path.isfile(srcpth): + dstpth = os.path.join(downloaded_bindir, target) + print(f"copying {srcpth} -> {dstpth}") + shutil.copy(srcpth, dstpth) + + +def test_rebuild_mf6_release(): + rebuild_mf6_release() + + +if __name__ == "__main__": + test_create_dirs() + test_getmfexes(verify=False) + test_rebuild_mf6_release() diff --git a/modflow_devtools/utilities/usgsprograms.py b/modflow_devtools/utilities/usgsprograms.py new file mode 100644 index 00000000..dde395b4 --- /dev/null +++ b/modflow_devtools/utilities/usgsprograms.py @@ -0,0 +1,533 @@ +"""Utility functions to extract information for a target from the USGS +application database. Available functionality includes: + +1. Get a list of available targets +2. Get data for a specific target +3. Get a dictionary with the data for all targets +4. Get the current version of a target +5. Get a list indicating if single and double precsion versions of the + target application should be built +6. Functions to load, update, and export a USGS-style "code.json" json file + containing information in the USGS application database + +A table listing the available pymake targets is included below: + +.. csv-table:: Available pymake targets + :file: ./usgsprograms.txt + :widths: 10, 10, 10, 20, 10, 10, 10, 10, 10 + :header-rows: 1 + +""" +import datetime +import json +import os +import sys + +from modflow_devtools.utilities.download import _request_header + + +class dotdict(dict): + """dot.notation access to dictionary attributes.""" + + __getattr__ = dict.get + __setattr__ = dict.__setitem__ + __delattr__ = dict.__delitem__ + + +# data file containing the USGS program data +program_data_file = "usgsprograms.txt" + +# keys to create for each target +target_keys = ( + "version", + "current", + "url", + "dirname", + "srcdir", + "standard_switch", + "double_switch", + "shared_object", + "url_download_asset_date", +) + + +def _str_to_bool(s): + """Convert "True" and "False" strings to a boolean. + + Parameters + ---------- + s : str + String representation of boolean + + Returns + ------- + + """ + if s == "True": + return True + elif s == "False": + return False + else: + msg = f'Invalid string passed - "{s}"' + raise ValueError(msg) + + +class usgs_program_data: + """USGS program database class.""" + + def __init__(self): + """USGS program database init.""" + self._program_dict = self._build_usgs_database() + + def _build_usgs_database(self): + """Build the USGS program database. + + Returns + ------- + + """ + # pth = os.path.dirname(os.path.abspath(pymake.__file__)) + pth = os.path.dirname(os.path.abspath(__file__)) + fpth = os.path.join(pth, program_data_file) + url_in = open(fpth, "r").read().split("\n") + + program_data = {} + for line in url_in[1:]: + # skip blank lines + if len(line.strip()) < 1: + continue + # parse comma separated line + t = [item.strip() for item in line.split(sep=",")] + # programmatically build a dictionary for each target + d = {} + for idx, key in enumerate(target_keys): + if key in ("url_download_asset_date",): + value = None + else: + value = t[idx + 1] + if key in ( + "current", + "standard_switch", + "double_switch", + "shared_object", + ): + value = _str_to_bool(value) + d[key] = value + + # make it possible to access each key with a dot (.) + d = dotdict(d) + program_data[t[0]] = d + + return dotdict(program_data) + + def _target_data(self, key): + """Get the dictionary for the target key. + + Parameters + ---------- + key : str + Program key (name) + + Returns + ------- + return : dict + dictionary with attributes for program key (name) + + """ + if key not in self._program_dict: + msg = f'"{key}" key does not exist. Available keys: ' + for idx, k in enumerate(self._program_dict.keys()): + if idx > 0: + msg += ", " + msg += f'"{k}"' + raise KeyError(msg) + return self._program_dict[key] + + def _target_keys(self, current=False): + """Get the target keys. + + Parameters + ---------- + current : bool + boolean indicating if only current program versions should be + returned. (default is False) + + Returns + ------- + keys : list + list containing program keys (names) + + """ + if current: + keys = [ + key + for key in self._program_dict.keys() + if self._program_dict[key].current + ] + else: + keys = list(self._program_dict.keys()) + return keys + + @staticmethod + def get_target(key): + """Get the dictionary for a specified target. + + Parameters + ---------- + key : str + Target USGS program that may have a path and an extension + + Returns + ------- + program_dict : dict + Dictionary with USGS program attributes for the specified key + + """ + # remove path and extension from key + key = os.path.basename(key) + if ( + key.endswith(".exe") + or key.endswith(".dll") + or key.endswith(".so") + or key.endswith(".dylib") + ): + key = os.path.splitext(key)[0] + + # return program attributes + return usgs_program_data()._target_data(key) + + @staticmethod + def get_keys(current=False): + """Get target keys from the USGS program database. + + Parameters + ---------- + current : bool + If False, all USGS program targets are listed. If True, + only USGS program targets that are defined as current are + listed. Default is False. + + Returns + ------- + keys : list + list of USGS program targets + + """ + + return usgs_program_data()._target_keys(current=current) + + @staticmethod + def get_program_dict(): + """Get the complete USGS program database. + + Returns + ------- + program_dict : dict + Dictionary with USGS program attributes for all targets + + """ + return usgs_program_data()._program_dict + + @staticmethod + def get_precision(key): + """Get the dictionary for a specified target. + + Parameters + ---------- + key : str + Target USGS program + + Returns + ------- + precision : list + List + + """ + target = usgs_program_data().get_target(key) + precision = [] + if target.standard_switch: + precision.append(False) + if target.double_switch: + precision.append(True) + return precision + + @staticmethod + def get_version(key): + """Get the current version of the specified target. + + Parameters + ---------- + key : str + Target USGS program + + Returns + ------- + version : str + current version of the specified target + + """ + target = usgs_program_data().get_target(key) + return target.version + + @staticmethod + def list_targets(current=False): + """Print a list of the available USGS program targets. + + Parameters + ---------- + current : bool + If False, all USGS program targets are listed. If True, + only USGS program targets that are defined as current are + listed. Default is False. + + Returns + ------- + + """ + targets = usgs_program_data()._target_keys(current=current) + targets.sort() + msg = "Available targets:\n" + for idx, target in enumerate(targets): + msg += f" {idx + 1:02d} {target}\n" + print(msg) + + return + + @staticmethod + def export_json( + fpth="code.json", + prog_data=None, + current=False, + update=True, + write_markdown=False, + verbose=False, + ): + """Export USGS program data as a json file. + + Parameters + ---------- + fpth : str + Path for the json file to be created. Default is "code.json" + prog_data : dict + User-specified program database. If prog_data is None, it will + be created from the USGS program database + current : bool + If False, all USGS program targets are listed. If True, + only USGS program targets that are defined as current are + listed. Default is False. + update : bool + If True, existing targets in the user-specified program database + with values in the USGS program database. If False, existing + targets in the user-specified program database will not be + updated. Default is True. + write_markdown : bool + If True, write markdown file that includes the target name, + version, and the last-modified date of the download asset (url). + Default is False. + verbose : bool + boolean for verbose output to terminal + + + Returns + ------- + + """ + # print a message + sel = "all of the" + if prog_data is not None: + sel = "select" + elif current: + sel = "the current" + print( + f'writing a json file ("{fpth}") ' + + f"of {sel} USGS programs\n" + + f'in the "{program_data_file}" database.' + ) + if prog_data is not None: + for idx, key in enumerate(prog_data.keys()): + print(f" {idx + 1:>2d}: {key}") + print("\n") + + # get usgs program data + udata = usgs_program_data.get_program_dict() + + # process the program data + if prog_data is None: + if current: + tdict = {} + for key, value in udata.items(): + if value.current: + tdict[key] = value + prog_data = tdict + # replace existing keys in prog_data with values from + # same key in usgs_program_data + else: + if update: + ukeys = usgs_program_data.get_keys() + pkeys = list(prog_data.keys()) + for key in pkeys: + if key in ukeys: + prog_data[key] = udata[key] + + # update the date of each asset if standard code.json object + for target, target_dict in prog_data.items(): + if "url" in target_dict.keys(): + url = target_dict["url"] + header = _request_header(url, verbose=verbose) + keys = list(header.headers.keys()) + for key in ("Last-Modified", "Date"): + if key in keys: + url_date = header.headers[key] + url_data_obj = datetime.datetime.strptime( + url_date, "%a, %d %b %Y %H:%M:%S %Z" + ) + datetime_obj_utc = url_data_obj.replace( + tzinfo=datetime.timezone.utc + ) + datetime_str = datetime_obj_utc.strftime("%m/%d/%Y") + prog_data[target][ + "url_download_asset_date" + ] = datetime_str + break + + # export file + try: + with open(fpth, "w") as f: + json.dump(prog_data, f, indent=4) + except: + msg = f'could not export json file "{fpth}"' + raise IOError(msg) + + # export code.json to --appdir directory, if the + # command line argument was specified. Only done if not CI + # command line argument was specified. Only done if not CI + appdir = "." + for idx, argv in enumerate(sys.argv): + if argv in ("--appdir", "-ad"): + appdir = sys.argv[idx + 1] + + # make appdir if it does not already exist + if not os.path.isdir(appdir): + os.makedirs(appdir) + + # write code.json + if appdir != ".": + dst = os.path.join(appdir, fpth) + with open(dst, "w") as f: + json.dump(prog_data, f, indent=4) + + # write code.md + if prog_data is not None and write_markdown: + file_obj = open("code.md", "w") + line = "| Program | Version | UTC Date |" + file_obj.write(line + "\n") + line = "| ------- | ------- | ---- |" + file_obj.write(line + "\n") + for target, target_dict in prog_data.items(): + keys = list(target_dict.keys()) + line = f"| {target} | {target_dict['version']} |" + date_key = "url_download_asset_date" + if date_key in keys: + line += f" {target_dict[date_key]} |" + else: + line += " |" + line += "\n" + file_obj.write(line) + file_obj.close() + + return + + @staticmethod + def load_json(fpth="code.json"): + """Load an existing code json file. Basic error checking is done to + make sure the file contains the correct keys. + + Parameters + ---------- + fpth : str + Path for the json file to be created. Default is "code.json" + + Returns + ------- + json_dict : dict + Valid USGS program database + + """ + try: + with open(fpth, "r") as f: + json_dict = json.load(f) + for key, value in json_dict.items(): + json_dict[key] = dotdict(value) + except: + json_dict = None + + # check that the json file has valid keys + msg = f'invalid json format in "{fpth}"' + if json_dict is not None: + for key, value in json_dict.items(): + try: + for kk in value.keys(): + if kk not in target_keys: + raise KeyError(msg + f' - key ("{kk}")') + except: + raise KeyError(msg) + + return json_dict + + @staticmethod + def list_json(fpth="code.json"): + """List an existing code json file. + + Parameters + ---------- + fpth : str + Path for the json file to be listed. Default is "code.json" + + Returns + ------- + + """ + json_dict = usgs_program_data.load_json(fpth) + + if json_dict is not None: + print(f'Data in "{fpth}"') + for key, value in json_dict.items(): + print(f" target: {key}") + for kkey, vvalue in value.items(): + print(f" {kkey}: {vvalue}") + else: + msg = f'could not load json file "{fpth}".' + raise IOError(msg) + + # print continuation line + print("\n") + + return + + @staticmethod + def update_json(fpth="code.json", temp_dict=None): + """UPDATE an existing code json file. + + Parameters + ---------- + fpth : str + Path for the json file to be listed. Default is "code.json" + + temp_dict : dict + Dictionary with USGS program data for a target + + Returns + ------- + + """ + if temp_dict is not None: + if os.path.isfile(fpth): + json_dict = usgs_program_data.load_json(fpth=fpth) + if json_dict is not None: + for key, value in temp_dict.items(): + if key not in list(json_dict.keys()): + json_dict[key] = value + temp_dict = json_dict + usgs_program_data.export_json(fpth, prog_data=temp_dict) + + return diff --git a/modflow_devtools/utilities/usgsprograms.txt b/modflow_devtools/utilities/usgsprograms.txt new file mode 100644 index 00000000..c0c50726 --- /dev/null +++ b/modflow_devtools/utilities/usgsprograms.txt @@ -0,0 +1,25 @@ +target , version, current, url , dirname , srcdir , standard_switch, double_switch, shared_object +mf6 , 6.3.0 , True , https://github.com/MODFLOW-USGS/modflow6/releases/download/6.3.0/mf6.3.0_linux.zip , mf6.3.0_linux , src , True , False , False +zbud6 , 6.3.0 , True , https://github.com/MODFLOW-USGS/modflow6/releases/download/6.3.0/mf6.3.0_linux.zip , mf6.3.0_linux , utils/zonebudget/src, True , False , False +libmf6 , 6.3.0 , True , https://github.com/MODFLOW-USGS/modflow6/releases/download/6.3.0/mf6.3.0_linux.zip , mf6.3.0_linux , srcbmi , True , False , True +mp7 , 7.2.001, True , https://water.usgs.gov/water-resources/software/MODPATH/modpath_7_2_001.zip , modpath_7_2_001 , source , True , False , False +mt3dms , 5.3.0 , True , https://hydro.geo.ua.edu/mt3d/mt3dms_530.exe , mt3dms5.3.0 , src/true-binary , True , False , False +mt3dusgs , 1.1.0 , True , https://water.usgs.gov/water-resources/software/MT3D-USGS/mt3dusgs1.1.0.zip , mt3dusgs1.1.0 , src , True , False , False +vs2dt , 3.3 , True , https://water.usgs.gov/water-resources/software/VS2DI/vs2dt3_3.zip , vs2dt3_3 , include , True , False , False +triangle , 1.6 , True , https://www.netlib.org/voronoi/triangle.zip , triangle1.6 , src , True , False , False +gridgen , 1.0.02 , True , https://water.usgs.gov/water-resources/software/GRIDGEN/gridgen.1.0.02.zip , gridgen.1.0.02 , src , True , False , False +crt , 1.3.1 , True , https://water.usgs.gov/ogw/CRT/CRT_1.3.1.zip , CRT_1.3.1 , SOURCE , True , False , False +gsflow , 2.2.0 , True , https://water.usgs.gov/water-resources/software/gsflow/gsflow_2.2.0_linux.zip , gsflow_2.2.0_linux , src , True , False , False +sutra , 3.0 , True , https://water.usgs.gov/water-resources/software/sutra/SUTRA_3_0_0.zip , SutraSuite , SUTRA_3_0/source , True , False , False +mf2000 , 1.19.01, True , https://water.usgs.gov/nrp/gwsoftware/modflow2000/mf2k1_19_01.tar.gz , mf2k.1_19 , src , True , False , False +mf2005 , 1.12.00, True , https://github.com/MODFLOW-USGS/mf2005/releases/download/v.1.12.00/MF2005.1_12u.zip , MF2005.1_12u , src , True , True , False +mf2005.1.11, 1.11.00, False , https://water.usgs.gov/ogw/modflow/archive-mf2005/MODFLOW-2005_v1.11.00/mf2005v1_11_00_unix.zip, Unix , src , True , False , False +mfusg , 1.5 , True , https://water.usgs.gov/water-resources/software/MODFLOW-USG/mfusg1_5.zip , mfusg1_5 , src , True , True , False +zonbudusg , 1.5 , True , https://water.usgs.gov/water-resources/software/MODFLOW-USG/mfusg1_5.zip , mfusg1_5 , src/zonebudusg , True , False , False +swtv4 , 4.00.05, True , https://water.usgs.gov/water-resources/software/SEAWAT/swt_v4_00_05.zip , swt_v4_00_05 , source , False , True , False +mp6 , 6.0.1 , True , https://water.usgs.gov/water-resources/software/MODPATH/modpath.6_0_01.zip , modpath.6_0 , src , True , False , False +mflgr , 2.0.0 , True , https://water.usgs.gov/ogw/modflow-lgr/modflow-lgr-v2.0.0/mflgrv2_0_00.zip , mflgr.2_0 , src , True , True , False +zonbud3 , 3.01 , True , https://water.usgs.gov/water-resources/software/ZONEBUDGET/zonbud3_01.exe , Zonbud.3_01 , Src , True , False , False +mfnwt1.1.4 , 1.1.4 , False , https://water.usgs.gov/water-resources/software/MODFLOW-NWT/MODFLOW-NWT_1.1.4.zip , MODFLOW-NWT_1.1.4 , src , True , False , False +mfnwt , 1.2.0 , True , https://water.usgs.gov/water-resources/software/MODFLOW-NWT/MODFLOW-NWT_1.2.0.zip , MODFLOW-NWT_1.2.0 , src , True , True , False +prms , 5.2.1 , True , https://water.usgs.gov/water-resources/software/PRMS/prms_5.2.1_linux.zip , prms_5.2.1_linux , src , True , False , False From 3ac39339405c3fd807b1570f9da669b09e9342c6 Mon Sep 17 00:00:00 2001 From: mjreno Date: Tue, 19 Jul 2022 16:11:59 -0400 Subject: [PATCH 2/6] setup: add download, dependencies and related scripts --- modflow_devtools/__init__.py | 16 +++++++++++----- modflow_devtools/utilities/get_exes.py | 2 +- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/modflow_devtools/__init__.py b/modflow_devtools/__init__.py index ce7c227b..fe6dbab3 100644 --- a/modflow_devtools/__init__.py +++ b/modflow_devtools/__init__.py @@ -2,6 +2,16 @@ development.""" +from .common_regression import ( + get_example_basedir, + get_example_dirs, + get_home_dir, + get_select_dirs, + get_select_packages, + is_directory_available, + set_mf6_regression, +) + # modflow_devtools from .config import ( __author__, @@ -12,10 +22,6 @@ __status__, __version__, ) -from .common_regression import get_example_basedir, get_example_dirs, \ - get_home_dir, get_select_dirs, \ - get_select_packages, is_directory_available, \ - set_mf6_regression from .framework import running_on_CI, set_teardown_test, testing_framework from .simulation import Simulation from .targets import get_mf6_version, get_target_dictionary, run_exe @@ -51,7 +57,6 @@ write_head, ) from .utilities.disu_util import get_disu_kwargs -from .utilities.usgsprograms import usgs_program_data from .utilities.download import ( download_and_unzip, get_repo_assets, @@ -60,6 +65,7 @@ repo_latest_version, zip_all, ) +from .utilities.usgsprograms import usgs_program_data # define public interfaces __all__ = [ diff --git a/modflow_devtools/utilities/get_exes.py b/modflow_devtools/utilities/get_exes.py index 7dfb8501..9c1aa757 100644 --- a/modflow_devtools/utilities/get_exes.py +++ b/modflow_devtools/utilities/get_exes.py @@ -3,10 +3,10 @@ import os import shutil +from build_exes import meson_build from download import download_and_unzip, getmfexes from usgsprograms import usgs_program_data -from build_exes import meson_build from modflow_devtools import running_on_CI if running_on_CI(): From 6dda02ce3e62b71b4188f39df23a76e7e2afec9b Mon Sep 17 00:00:00 2001 From: mjreno Date: Tue, 19 Jul 2022 17:12:56 -0400 Subject: [PATCH 3/6] setup: add download, dependencies and related scripts --- .pylintrc | 2 ++ modflow_devtools/utilities/download.py | 2 +- modflow_devtools/utilities/get_exes.py | 8 ++++---- 3 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 .pylintrc diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 00000000..dab3d6ee --- /dev/null +++ b/.pylintrc @@ -0,0 +1,2 @@ +[MASTER] +ignore=download.py diff --git a/modflow_devtools/utilities/download.py b/modflow_devtools/utilities/download.py index d613d1f6..6b0a63c3 100644 --- a/modflow_devtools/utilities/download.py +++ b/modflow_devtools/utilities/download.py @@ -216,7 +216,7 @@ def _request_get(url, verify=True, timeout=1, max_requests=10, verbose=False): else: msg = "Cannot open request from:\n" + f" {url}\n\n" print(msg) - req.raise_for_status() + raise requests.HTTPError(msg) # successful request break diff --git a/modflow_devtools/utilities/get_exes.py b/modflow_devtools/utilities/get_exes.py index 9c1aa757..362960c2 100644 --- a/modflow_devtools/utilities/get_exes.py +++ b/modflow_devtools/utilities/get_exes.py @@ -3,12 +3,12 @@ import os import shutil -from build_exes import meson_build -from download import download_and_unzip, getmfexes -from usgsprograms import usgs_program_data - from modflow_devtools import running_on_CI +from .build_exes import meson_build +from .download import download_and_unzip, getmfexes +from .usgsprograms import usgs_program_data + if running_on_CI(): print("running on CI environment") os.environ["PYMAKE_DOUBLE"] = "1" From 7ff957bdb3ee4d29387add7f36cb74114dd66d9b Mon Sep 17 00:00:00 2001 From: mjreno Date: Tue, 19 Jul 2022 18:36:14 -0400 Subject: [PATCH 4/6] setup: add download, dependencies and related scripts --- .pylintrc | 2 -- setup.cfg | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 .pylintrc diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index dab3d6ee..00000000 --- a/.pylintrc +++ /dev/null @@ -1,2 +0,0 @@ -[MASTER] -ignore=download.py diff --git a/setup.cfg b/setup.cfg index e792eb67..782f22fc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,6 +33,7 @@ packages = find: python_requires = >=3.7 install_requires = numpy + requests flopy [flake8] From d883f67ce915649f84192412ef9d32b7df9cef3e Mon Sep 17 00:00:00 2001 From: mjreno Date: Tue, 19 Jul 2022 18:41:34 -0400 Subject: [PATCH 5/6] setup: add download, dependencies and related scripts --- .github/workflows/modflow-devtools-linting-install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/modflow-devtools-linting-install.yml b/.github/workflows/modflow-devtools-linting-install.yml index e9346ea1..deb76dbf 100644 --- a/.github/workflows/modflow-devtools-linting-install.yml +++ b/.github/workflows/modflow-devtools-linting-install.yml @@ -28,7 +28,7 @@ jobs: - name: Install packages run: | - pip install numpy flopy pylint flake8 black + pip install numpy flopy pylint flake8 black requests - name: Run isort run: | From a1ad4bef2890d573e0c101a02aa9b9abfd841075 Mon Sep 17 00:00:00 2001 From: mjreno Date: Wed, 20 Jul 2022 08:10:54 -0400 Subject: [PATCH 6/6] setup: add download, dependencies and related scripts --- modflow_devtools/utilities/download.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modflow_devtools/utilities/download.py b/modflow_devtools/utilities/download.py index 6b0a63c3..7e24c88b 100644 --- a/modflow_devtools/utilities/download.py +++ b/modflow_devtools/utilities/download.py @@ -1,3 +1,5 @@ +# pylint: disable=E1101 + """Utility functions to: 1. download and unzip software releases from the USGS and other organizations