From 76da53085bb7ace1357f12f753e1f5cffe177438 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 8 Apr 2020 09:08:12 -0600 Subject: [PATCH 1/8] Move soma/Xkm/32toXkm test cases to "broken" This merge moves soma/4km/32to4km and soma/8km/32to8km test cases into a new subdirectory called "broken" since these test cases are not working and won't be fixed anytime soon. With this change, `./list_testcases.py` and `./setup_testcases.py` won't pick up these tests because their driver config files aren't at the expected directory level. --- .../compass/ocean/soma/4km/{ => broken}/32to4km/config_driver.xml | 0 .../ocean/soma/4km/{ => broken}/32to4km/config_forward.xml | 0 .../compass/ocean/soma/4km/{ => broken}/32to4km/config_init1.xml | 0 .../compass/ocean/soma/4km/{ => broken}/32to4km/config_init2.xml | 0 .../compass/ocean/soma/8km/{ => broken}/32to8km/config_driver.xml | 0 .../ocean/soma/8km/{ => broken}/32to8km/config_forward.xml | 0 .../compass/ocean/soma/8km/{ => broken}/32to8km/config_init1.xml | 0 .../compass/ocean/soma/8km/{ => broken}/32to8km/config_init2.xml | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename testing_and_setup/compass/ocean/soma/4km/{ => broken}/32to4km/config_driver.xml (100%) rename testing_and_setup/compass/ocean/soma/4km/{ => broken}/32to4km/config_forward.xml (100%) rename testing_and_setup/compass/ocean/soma/4km/{ => broken}/32to4km/config_init1.xml (100%) rename testing_and_setup/compass/ocean/soma/4km/{ => broken}/32to4km/config_init2.xml (100%) rename testing_and_setup/compass/ocean/soma/8km/{ => broken}/32to8km/config_driver.xml (100%) rename testing_and_setup/compass/ocean/soma/8km/{ => broken}/32to8km/config_forward.xml (100%) rename testing_and_setup/compass/ocean/soma/8km/{ => broken}/32to8km/config_init1.xml (100%) rename testing_and_setup/compass/ocean/soma/8km/{ => broken}/32to8km/config_init2.xml (100%) diff --git a/testing_and_setup/compass/ocean/soma/4km/32to4km/config_driver.xml b/testing_and_setup/compass/ocean/soma/4km/broken/32to4km/config_driver.xml similarity index 100% rename from testing_and_setup/compass/ocean/soma/4km/32to4km/config_driver.xml rename to testing_and_setup/compass/ocean/soma/4km/broken/32to4km/config_driver.xml diff --git a/testing_and_setup/compass/ocean/soma/4km/32to4km/config_forward.xml b/testing_and_setup/compass/ocean/soma/4km/broken/32to4km/config_forward.xml similarity index 100% rename from testing_and_setup/compass/ocean/soma/4km/32to4km/config_forward.xml rename to testing_and_setup/compass/ocean/soma/4km/broken/32to4km/config_forward.xml diff --git a/testing_and_setup/compass/ocean/soma/4km/32to4km/config_init1.xml b/testing_and_setup/compass/ocean/soma/4km/broken/32to4km/config_init1.xml similarity index 100% rename from testing_and_setup/compass/ocean/soma/4km/32to4km/config_init1.xml rename to testing_and_setup/compass/ocean/soma/4km/broken/32to4km/config_init1.xml diff --git a/testing_and_setup/compass/ocean/soma/4km/32to4km/config_init2.xml b/testing_and_setup/compass/ocean/soma/4km/broken/32to4km/config_init2.xml similarity index 100% rename from testing_and_setup/compass/ocean/soma/4km/32to4km/config_init2.xml rename to testing_and_setup/compass/ocean/soma/4km/broken/32to4km/config_init2.xml diff --git a/testing_and_setup/compass/ocean/soma/8km/32to8km/config_driver.xml b/testing_and_setup/compass/ocean/soma/8km/broken/32to8km/config_driver.xml similarity index 100% rename from testing_and_setup/compass/ocean/soma/8km/32to8km/config_driver.xml rename to testing_and_setup/compass/ocean/soma/8km/broken/32to8km/config_driver.xml diff --git a/testing_and_setup/compass/ocean/soma/8km/32to8km/config_forward.xml b/testing_and_setup/compass/ocean/soma/8km/broken/32to8km/config_forward.xml similarity index 100% rename from testing_and_setup/compass/ocean/soma/8km/32to8km/config_forward.xml rename to testing_and_setup/compass/ocean/soma/8km/broken/32to8km/config_forward.xml diff --git a/testing_and_setup/compass/ocean/soma/8km/32to8km/config_init1.xml b/testing_and_setup/compass/ocean/soma/8km/broken/32to8km/config_init1.xml similarity index 100% rename from testing_and_setup/compass/ocean/soma/8km/32to8km/config_init1.xml rename to testing_and_setup/compass/ocean/soma/8km/broken/32to8km/config_init1.xml diff --git a/testing_and_setup/compass/ocean/soma/8km/32to8km/config_init2.xml b/testing_and_setup/compass/ocean/soma/8km/broken/32to8km/config_init2.xml similarity index 100% rename from testing_and_setup/compass/ocean/soma/8km/32to8km/config_init2.xml rename to testing_and_setup/compass/ocean/soma/8km/broken/32to8km/config_init2.xml From 925da04750d1ea1a68ae925b463f319cb66ee351 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 8 Apr 2020 09:11:28 -0600 Subject: [PATCH 2/8] Add missing define_vertical_trid.py links The Maine, QU60 and SOQU60to15 test cases now have the links to the python script for defining their vertical grids that they need to be set up successfully. --- .../compass/ocean/coastal/Maine/init/define_vertical_grid.py | 1 + .../compass/ocean/global_ocean/QU60/init/define_vertical_grid.py | 1 + .../ocean/global_ocean/SOQU60to15/init/define_vertical_grid.py | 1 + 3 files changed, 3 insertions(+) create mode 120000 testing_and_setup/compass/ocean/coastal/Maine/init/define_vertical_grid.py create mode 120000 testing_and_setup/compass/ocean/global_ocean/QU60/init/define_vertical_grid.py create mode 120000 testing_and_setup/compass/ocean/global_ocean/SOQU60to15/init/define_vertical_grid.py diff --git a/testing_and_setup/compass/ocean/coastal/Maine/init/define_vertical_grid.py b/testing_and_setup/compass/ocean/coastal/Maine/init/define_vertical_grid.py new file mode 120000 index 0000000000..4e1cac8e2b --- /dev/null +++ b/testing_and_setup/compass/ocean/coastal/Maine/init/define_vertical_grid.py @@ -0,0 +1 @@ +../../../scripts/vertical_grid/define_vertical_grid_64_layers.py \ No newline at end of file diff --git a/testing_and_setup/compass/ocean/global_ocean/QU60/init/define_vertical_grid.py b/testing_and_setup/compass/ocean/global_ocean/QU60/init/define_vertical_grid.py new file mode 120000 index 0000000000..4e1cac8e2b --- /dev/null +++ b/testing_and_setup/compass/ocean/global_ocean/QU60/init/define_vertical_grid.py @@ -0,0 +1 @@ +../../../scripts/vertical_grid/define_vertical_grid_64_layers.py \ No newline at end of file diff --git a/testing_and_setup/compass/ocean/global_ocean/SOQU60to15/init/define_vertical_grid.py b/testing_and_setup/compass/ocean/global_ocean/SOQU60to15/init/define_vertical_grid.py new file mode 120000 index 0000000000..4e1cac8e2b --- /dev/null +++ b/testing_and_setup/compass/ocean/global_ocean/SOQU60to15/init/define_vertical_grid.py @@ -0,0 +1 @@ +../../../scripts/vertical_grid/define_vertical_grid_64_layers.py \ No newline at end of file From 933ec422cbac462b7f7ed695fc33d1f803eba1cf Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 8 Apr 2020 09:15:40 -0600 Subject: [PATCH 3/8] Use conda env gpmetis for surface_waves/thickness_source/1km We no longer define a path to metis in the config file, so the version from the conda environment needs to be used instead. --- .../surface_waves/thickness_source/1km/config_forward.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/testing_and_setup/compass/ocean/surface_waves/thickness_source/1km/config_forward.xml b/testing_and_setup/compass/ocean/surface_waves/thickness_source/1km/config_forward.xml index b228d40b3a..a387388d27 100644 --- a/testing_and_setup/compass/ocean/surface_waves/thickness_source/1km/config_forward.xml +++ b/testing_and_setup/compass/ocean/surface_waves/thickness_source/1km/config_forward.xml @@ -6,7 +6,6 @@ - @@ -63,7 +62,7 @@ init.nc - + 4 From 82170974629fbf5e54b5f0b9af9510440ba5ac0e Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Fri, 20 Mar 2020 16:24:33 +0100 Subject: [PATCH 4/8] Detect MPI from a conda environment setup_testcase.py Add support for a "conda_mpi" attribute to "step" tags. If this attribute is set to "true" and MPI is present in the conda environment, that command will be called with `mpirun` from the conda envrionment. This is needed to support compass conda enviornments with mpich. Python scripts and modules that use the netcdf4 package with mpich support don't work properly on many compute nodes (e.g. Grizzly at LANL and Anvil at ANL) unless they are prefixed with `mpirun -np 1` --- testing_and_setup/compass/setup_testcase.py | 45 +++++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/testing_and_setup/compass/setup_testcase.py b/testing_and_setup/compass/setup_testcase.py index a4553301e6..cd2d9518f4 100755 --- a/testing_and_setup/compass/setup_testcase.py +++ b/testing_and_setup/compass/setup_testcase.py @@ -591,7 +591,7 @@ def generate_driver_scripts(config_file, configs): # {{{ # Process tags if grandchild.tag == 'step': process_script_step(grandchild, configs, ' ', - script) + script, conda_mpi=False) # Process tags elif grandchild.tag == 'define_env_var': process_env_define_step(grandchild, configs, ' ', @@ -661,7 +661,8 @@ def process_env_define_step(var_tag, configs, indentation, script_file): # {{{ # }}} -def process_script_step(step, configs, indentation, script_file): # {{{ +def process_script_step(step, configs, indentation, script_file, + conda_mpi=None): # {{{ # Determine step attributes. if 'executable_name' in step.attrib.keys() and 'executable' in \ step.attrib.keys(): @@ -697,6 +698,15 @@ def process_script_step(step, configs, indentation, script_file): # {{{ except KeyError: executable = step.attrib['executable'] + if 'conda_mpi' in step.attrib: + # This step explicitly asks says whether to use conda MPI + conda_mpi = step.attrib['conda_mpi'] == "true" + elif conda_mpi is None: + # if not set explicitly, we will try to use conda MPI for executables + # start with "python" or end with ".py" + basename = os.path.basename(executable) + conda_mpi = (basename.startswith('python') or basename.endswith('.py')) + # Write step header script_file.write("\n") @@ -708,7 +718,7 @@ def process_script_step(step, configs, indentation, script_file): # {{{ script_file.write("{}# Run command is:\n".format(indentation)) - command_args = [executable] + command_args = add_executable_prefix(executable, configs, conda_mpi) # Process step arguments for argument in step: if argument.tag == 'argument': @@ -877,6 +887,10 @@ def process_field_definition(field_tag, configs, script, file1, file2, command_args = [compare_executable, '-q', '-1', file1, '-2', file2, '-v', field_name] + if configs.getboolean('conda', 'use_conda_mpi'): + prefix = configs.get('conda', 'python_prefix').split(' ') + command_args = prefix + command_args + # Determine norm thresholds if baseline_comp: command_args.extend(['--l1', '0.0', '--l2', '0.0', '--linf', '0.0']) @@ -1549,6 +1563,19 @@ def get_case_name(config_file): # {{{ return name # }}} + + +def add_executable_prefix(executable, configs, conda_mpi): # {{{ + """ + Prepend a prefix to the executable if conda MPI is requested and MPI is + present in the conda environment + """ + command_args = [executable] + if conda_mpi and configs.getboolean('conda', 'use_conda_mpi'): + prefix = configs.get('conda', 'python_prefix').split(' ') + command_args = prefix + command_args + return command_args +# }}} # }}} @@ -1678,6 +1705,18 @@ def get_case_name(config_file): # {{{ config.set('script_input_arguments', 'model_runtime', args.model_runtime) + if not config.has_section('conda'): + config.add_section('conda') + + config.set('conda', 'use_conda_mpi', 'False') + if 'CONDA_PREFIX' in os.environ: + # We're running from a conda environment + mpirun_path = '{}/bin/mpirun'.format(os.environ['CONDA_PREFIX']) + if os.path.exists(mpirun_path): + # MPI is installed in that conda environment + config.set('conda', 'use_conda_mpi', 'True') + config.set('conda', 'python_prefix', '{} -np 1'.format(mpirun_path)) + # Build variables for history output old_dir = os.getcwd() os.chdir(config.get('script_paths', 'script_path')) From 714725f6631324e824253794d342f376b910e1e9 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 19 Mar 2020 16:04:01 -0500 Subject: [PATCH 5/8] Update scripts to remove some subprocess calls The paraview extractor can now be called as a function rather than a script, and this is done during base-mesh generation and culling. SCRIP files can now also be created with a function, so a script call is replaced with a function here as well. With these changes, calls to python scripts that use NetCDF in the parallel conda enviornment will now work as long as they are called with `mpirun -np 1` --- .../scripts/create_E3SM_coupling_files.py | 23 +++++++-------- .../ocean/global_ocean/scripts/cull_mesh.py | 29 +++++++------------ .../ocean/jigsaw_to_MPAS/build_mesh.py | 14 +++------ 3 files changed, 25 insertions(+), 41 deletions(-) diff --git a/testing_and_setup/compass/ocean/global_ocean/scripts/create_E3SM_coupling_files.py b/testing_and_setup/compass/ocean/global_ocean/scripts/create_E3SM_coupling_files.py index b4e09dc34c..fafb2c8a8f 100755 --- a/testing_and_setup/compass/ocean/global_ocean/scripts/create_E3SM_coupling_files.py +++ b/testing_and_setup/compass/ocean/global_ocean/scripts/create_E3SM_coupling_files.py @@ -21,9 +21,10 @@ from mpas_tools.ocean.moc import make_moc_basins_and_transects from mpas_tools.io import write_netcdf import mpas_tools.conversion +import pyproj from pyremap import MpasMeshDescriptor, ProjectionGridDescriptor, Remapper, \ get_lat_lon_descriptor, get_polar_descriptor -import pyproj +from mpas_tools.scrip.from_mpas import scrip_from_mpas # }}} @@ -248,25 +249,19 @@ def scrip(config): # {{{ nomaskStr = '' # create links - make_link('../mesh.nc', '{}.nc'.format(mesh_name)) + mesh_file = '{}.nc'.format(mesh_name) + make_link('../mesh.nc', mesh_file) # command line execution scrip_file = 'ocean.{}{}.scrip.{}.nc'.format(mesh_name, nomaskStr, date_string) - args = ['create_SCRIP_file_from_MPAS_mesh.py', - '-m', '{}.nc'.format(mesh_name), - '-s', scrip_file] - run_command(args) + scrip_from_mpas(mesh_file, scrip_file) if ice_shelf_cavities: scrip_file_mask = 'ocean.{}.mask.scrip.{}.nc'.format(mesh_name, date_string) - args = ['create_SCRIP_file_from_MPAS_mesh.py', - '-m', mesh_name + '.nc', - '-s', scrip_file_mask, - '--landice'] - run_command(args) + scrip_from_mpas(mesh_file, scrip_file, useLandIceMask=True) # make links to output directories os.chdir('../assembled_files_for_upload/inputdata/ocn/mpas-o/{}'.format( @@ -1001,7 +996,8 @@ def make_analysis_lat_lon_map(config, mesh_name): remapper = Remapper(inDescriptor, outDescriptor, mappingFileName) mpiTasks = config.getint('main', 'nprocs') - remapper.build_mapping_file(method='bilinear', mpiTasks=mpiTasks) + remapper.build_mapping_file(method='bilinear', mpiTasks=mpiTasks, + tempdir='.') # }}} @@ -1033,7 +1029,8 @@ def make_analysis_polar_map(config, mesh_name, projection): remapper = Remapper(inDescriptor, outDescriptor, mappingFileName) mpiTasks = config.getint('main', 'nprocs') - remapper.build_mapping_file(method='bilinear', mpiTasks=mpiTasks) + remapper.build_mapping_file(method='bilinear', mpiTasks=mpiTasks, + tempdir='.') # }}} diff --git a/testing_and_setup/compass/ocean/global_ocean/scripts/cull_mesh.py b/testing_and_setup/compass/ocean/global_ocean/scripts/cull_mesh.py index 1503353842..4a7cce1d50 100755 --- a/testing_and_setup/compass/ocean/global_ocean/scripts/cull_mesh.py +++ b/testing_and_setup/compass/ocean/global_ocean/scripts/cull_mesh.py @@ -25,7 +25,6 @@ unicode_literals import os -import subprocess from optparse import OptionParser import xarray @@ -34,6 +33,7 @@ from mpas_tools.io import write_netcdf from mpas_tools.ocean.coastline_alteration import widen_transect_edge_masks, \ add_critical_land_blockages, add_land_locked_cells_to_mask +from mpas_tools.viz.paraview_extractor import extract_vtk parser = OptionParser() @@ -158,20 +158,13 @@ write_netcdf(dsLandIceCulledMesh, 'no_ISC_culled_mesh.nc', format=netcdfFormat) -args = ['paraview_vtk_field_extractor.py', - '--ignore_time', - '-d', 'maxEdges=', - '-v', 'allOnCells', - '-f', 'culled_mesh.nc', - '-o', 'culled_mesh_vtk'] -print("running", ' '.join(args)) -subprocess.check_call(args, env=os.environ.copy()) - -args = ['paraview_vtk_field_extractor.py', - '--ignore_time', - '-d', 'maxEdges=', - '-v', 'allOnCells', - '-f', 'no_ISC_culled_mesh.nc', - '-o', 'no_ISC_culled_mesh_vtk'] -print("running", ' '.join(args)) -subprocess.check_call(args, env=os.environ.copy()) +extract_vtk(ignore_time=True, dimension_list=['maxEdges='], + variable_list=['allOnCells'], filename_pattern='culled_mesh.nc', + out_dir='culled_mesh_vtk') + +if options.with_cavities: + extract_vtk(ignore_time=True, dimension_list=['maxEdges='], + variable_list=['allOnCells'], + filename_pattern='no_ISC_culled_mesh.nc', + out_dir='no_ISC_culled_mesh_vtk') + diff --git a/testing_and_setup/compass/ocean/jigsaw_to_MPAS/build_mesh.py b/testing_and_setup/compass/ocean/jigsaw_to_MPAS/build_mesh.py index 26d3410361..8d298a1193 100755 --- a/testing_and_setup/compass/ocean/jigsaw_to_MPAS/build_mesh.py +++ b/testing_and_setup/compass/ocean/jigsaw_to_MPAS/build_mesh.py @@ -12,7 +12,6 @@ from __future__ import absolute_import, division, print_function, \ unicode_literals -import subprocess import os import xarray import argparse @@ -20,6 +19,7 @@ from mpas_tools.conversion import convert from mpas_tools.io import write_netcdf +from mpas_tools.viz.paraview_extractor import extract_vtk from jigsaw_to_MPAS.jigsaw_driver import jigsaw_driver from jigsaw_to_MPAS.triangle_jigsaw_to_netcdf import jigsaw_to_netcdf @@ -121,15 +121,9 @@ def build_mesh( floodplain_elevation=floodplain_elevation) print('Step 8. Create vtk file for visualization') - args = ['paraview_vtk_field_extractor.py', - '--ignore_time', - '-l', - '-d', 'maxEdges=0', - '-v', 'allOnCells', - '-f', 'base_mesh.nc', - '-o', 'base_mesh_vtk'] - print("running", ' '.join(args)) - subprocess.check_call(args, env=os.environ.copy()) + extract_vtk(ignore_time=True, lonlat=True, dimension_list=['maxEdges='], + variable_list=['allOnCells'], filename_pattern='base_mesh.nc', + out_dir='base_mesh_vtk') print("***********************************************") print("** The global mesh file is base_mesh.nc **") From 9f0bb708d2aa4a58844b4468f7f8f47d9bb323bb Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Wed, 25 Mar 2020 12:43:13 +0100 Subject: [PATCH 6/8] Update compass environment version Also rename the load script for convenience. --- testing_and_setup/compass/README_ocean.md | 4 ++-- .../compass/{load_compass_env.bash => load_compass_env.sh} | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename testing_and_setup/compass/{load_compass_env.bash => load_compass_env.sh} (98%) diff --git a/testing_and_setup/compass/README_ocean.md b/testing_and_setup/compass/README_ocean.md index c49ad9e81e..be8ceb6304 100644 --- a/testing_and_setup/compass/README_ocean.md +++ b/testing_and_setup/compass/README_ocean.md @@ -6,11 +6,11 @@ To set up and run ocean test cases from COMPASS, you will need a conda environment. First, install Miniconda3 (if miniconda is not already installed), then create a new conda environment as follows: ``` bash -conda create -n compass_0.1.2 -c conda-forge -c e3sm python=3.7 compass=0.1.2 +conda create -n compass_0.1.3 -c conda-forge -c e3sm python=3.7 compass=0.1.3 ``` Each time you want to work with COMPASS, you will need to run: ``` -conda activate compass_0.1.2 +conda activate compass_0.1.3 ``` An appropriate conda environment is already available on Los Alamos National diff --git a/testing_and_setup/compass/load_compass_env.bash b/testing_and_setup/compass/load_compass_env.sh similarity index 98% rename from testing_and_setup/compass/load_compass_env.bash rename to testing_and_setup/compass/load_compass_env.sh index 5535977b5d..a60d03edd2 100644 --- a/testing_and_setup/compass/load_compass_env.bash +++ b/testing_and_setup/compass/load_compass_env.sh @@ -1,4 +1,4 @@ -version=0.1.2 +version=0.1.3 # The rest of the script should not need to be modified if [[ $HOSTNAME = "cori"* ]] || [[ $HOSTNAME = "dtn"* ]]; then From 566390a98e482071b318383205487c092453bc77 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 26 Mar 2020 16:11:15 +0100 Subject: [PATCH 7/8] Add full mpirun path to ESMF_RegridWeightGen calls This will make sure the compatible version of MPI gets used. --- .../scripts/create_E3SM_coupling_files.py | 37 ++++++++++++++----- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/testing_and_setup/compass/ocean/global_ocean/scripts/create_E3SM_coupling_files.py b/testing_and_setup/compass/ocean/global_ocean/scripts/create_E3SM_coupling_files.py index fafb2c8a8f..c0d1afc10f 100755 --- a/testing_and_setup/compass/ocean/global_ocean/scripts/create_E3SM_coupling_files.py +++ b/testing_and_setup/compass/ocean/global_ocean/scripts/create_E3SM_coupling_files.py @@ -386,30 +386,38 @@ def mapping(config, atm_scrip_tag): # {{{ atm_scrip_file = '{}.nc'.format(atm_scrip_tag) make_link('{}/{}'.format(atm_scrip_path, atm_scrip_file), atm_scrip_file) + if 'CONDA_PREFIX' not in os.environ: + raise ValueError('A COMPASS conda environment needs to be loaded.') + mpirun_path = '{}/bin/mpirun'.format(os.environ['CONDA_PREFIX']) + if os.path.exists(mpirun_path): + prefix = [mpirun_path, '-n', nprocs] + else: + prefix = list() + for method, short in [['conserve', 'aave'], ['bilinear', 'blin'], ['patch', 'patc']]: # Ocean to atmosphere mapping_file = 'map_{}{}_TO_{}_{}.{}.nc'.format( mesh_name, nomaskStr, atm_scrip_tag, short, date_string) - args = ['mpirun', '-n', nprocs, 'ESMF_RegridWeightGen', + args = ['ESMF_RegridWeightGen', '--method', method, '--source', ocn_scrip_file, '--destination', atm_scrip_file, '--weight', mapping_file, '--ignore_unmapped'] - run_command(args) + run_command(prefix + args) # Atmosphere to ocean mapping_file = 'map_{}_TO_{}{}_{}.{}.nc'.format( atm_scrip_tag, mesh_name, nomaskStr, short, date_string) - args = ['mpirun', '-n', nprocs, 'ESMF_RegridWeightGen', + args = ['ESMF_RegridWeightGen', '--method', method, '--source', atm_scrip_file, '--destination', ocn_scrip_file, '--weight', mapping_file, '--ignore_unmapped'] - run_command(args) + run_command(prefix + args) if ice_shelf_cavities: print("\n Mapping files with masks for ice shelf cavities") @@ -424,24 +432,25 @@ def mapping(config, atm_scrip_tag): # {{{ # Ocean to atmosphere mapping_file = 'map_{}.mask_TO_{}_{}.{}.nc'.format( mesh_name, atm_scrip_tag, short, date_string) - args = ['mpirun', '-n', nprocs, 'ESMF_RegridWeightGen', + args = ['ESMF_RegridWeightGen', '--method', method, '--source', ocn_scrip_file, '--destination', atm_scrip_file, '--weight', mapping_file, '--ignore_unmapped'] - run_command(args) + run_command(prefix + args) # Atmosphere to ocean mapping_file = 'map_{}_TO_{}.mask_{}.{}.nc'.format( atm_scrip_tag, mesh_name, short, date_string) - args = ['mpirun', '-n', nprocs, 'ESMF_RegridWeightGen', + args = ['ESMF_RegridWeightGen', '--method', method, '--source', atm_scrip_file, '--destination', ocn_scrip_file, '--weight', mapping_file, '--ignore_unmapped'] - run_command(args) + run_command(prefix + args) + # }}} @@ -598,6 +607,14 @@ def salinity_restoring(config): # {{{ date_string) make_link('../scrip/{}'.format(ocn_scrip_file), ocn_scrip_file) + if 'CONDA_PREFIX' not in os.environ: + raise ValueError('A COMPASS conda environment needs to be loaded.') + mpirun_path = '{}/bin/mpirun'.format(os.environ['CONDA_PREFIX']) + if os.path.exists(mpirun_path): + prefix = [mpirun_path, '-n', nprocs] + else: + prefix = list() + # execute commands salinity_restoring_output_file = \ 'sss.PHC2_monthlyClimatology.{}.{}.nc'.format(mesh_name, date_string) @@ -607,13 +624,13 @@ def salinity_restoring(config): # {{{ # mapping file, 1x1 to ocean mesh map_Levitus_file = 'map_Levitus_1x1_TO_{}_{}.{}.nc'.format( mesh_name, short, date_string) - args = ['mpirun', '-n', nprocs, 'ESMF_RegridWeightGen', + args = ['ESMF_RegridWeightGen', '--method', method, '--source', 'grid_Levitus_1x1_scrip_file.nc', '--destination', ocn_scrip_file, '--weight', map_Levitus_file, '--ignore_unmapped'] - run_command(args) + run_command(prefix + args) # remap from 1x1 to model grid args = ['ncremap', From 190338d0c2ea6b385746fcdf2133ef1e363c323a Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 9 Apr 2020 02:10:20 -0600 Subject: [PATCH 8/8] Add mpi_conda to planar_hex calls Since we can't detect automatically that this is a python script, (and that it needs to support compass mpi) we need to say so explicitly --- .../ocean/baroclinic_channel/10km/rpe_test/config_mpas_mesh.xml | 2 +- .../ocean/baroclinic_channel/1km/rpe_test/config_mpas_mesh.xml | 2 +- .../ocean/baroclinic_channel/4km/rpe_test/config_mpas_mesh.xml | 2 +- .../compass/ocean/dam_break/default/004m/config_init1.xml | 2 +- .../compass/ocean/dam_break/default/012m/config_init1.xml | 2 +- .../compass/ocean/drying_slope/meshes/1km/config_init.xml | 2 +- .../compass/ocean/drying_slope/meshes/250m/config_init.xml | 2 +- .../ocean/internal_waves/5km/rpe_test/config_mpas_mesh.xml | 2 +- .../compass/ocean/lock_exchange/0.5km/default/config_init1.xml | 2 +- .../ocean/lock_exchange/0.5km/rpe_test/config_mpas_mesh.xml | 2 +- .../compass/ocean/overflow/1km/rpe_test/config_mpas_mesh.xml | 2 +- .../compass/ocean/surface_waves/direct/1km/config_init1.xml | 2 +- .../ocean/surface_waves/thickness_source/1km/config_init1.xml | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/testing_and_setup/compass/ocean/baroclinic_channel/10km/rpe_test/config_mpas_mesh.xml b/testing_and_setup/compass/ocean/baroclinic_channel/10km/rpe_test/config_mpas_mesh.xml index 12979e1fd6..69865c8882 100644 --- a/testing_and_setup/compass/ocean/baroclinic_channel/10km/rpe_test/config_mpas_mesh.xml +++ b/testing_and_setup/compass/ocean/baroclinic_channel/10km/rpe_test/config_mpas_mesh.xml @@ -1,7 +1,7 @@ - + 16 50 10000.0 diff --git a/testing_and_setup/compass/ocean/baroclinic_channel/1km/rpe_test/config_mpas_mesh.xml b/testing_and_setup/compass/ocean/baroclinic_channel/1km/rpe_test/config_mpas_mesh.xml index 4f4c3b7fb7..90c06ce4f8 100644 --- a/testing_and_setup/compass/ocean/baroclinic_channel/1km/rpe_test/config_mpas_mesh.xml +++ b/testing_and_setup/compass/ocean/baroclinic_channel/1km/rpe_test/config_mpas_mesh.xml @@ -1,7 +1,7 @@ - + 160 500 1000.0 diff --git a/testing_and_setup/compass/ocean/baroclinic_channel/4km/rpe_test/config_mpas_mesh.xml b/testing_and_setup/compass/ocean/baroclinic_channel/4km/rpe_test/config_mpas_mesh.xml index 1720b75944..2a4a6d2cf2 100644 --- a/testing_and_setup/compass/ocean/baroclinic_channel/4km/rpe_test/config_mpas_mesh.xml +++ b/testing_and_setup/compass/ocean/baroclinic_channel/4km/rpe_test/config_mpas_mesh.xml @@ -1,7 +1,7 @@ - + 40 126 4000.0 diff --git a/testing_and_setup/compass/ocean/dam_break/default/004m/config_init1.xml b/testing_and_setup/compass/ocean/dam_break/default/004m/config_init1.xml index 4f8ca81a58..320f42365e 100644 --- a/testing_and_setup/compass/ocean/dam_break/default/004m/config_init1.xml +++ b/testing_and_setup/compass/ocean/dam_break/default/004m/config_init1.xml @@ -42,7 +42,7 @@ - + 325 700 0.04 diff --git a/testing_and_setup/compass/ocean/dam_break/default/012m/config_init1.xml b/testing_and_setup/compass/ocean/dam_break/default/012m/config_init1.xml index 7930342517..14e2e2280f 100644 --- a/testing_and_setup/compass/ocean/dam_break/default/012m/config_init1.xml +++ b/testing_and_setup/compass/ocean/dam_break/default/012m/config_init1.xml @@ -41,7 +41,7 @@ - + 108 232 0.12 diff --git a/testing_and_setup/compass/ocean/drying_slope/meshes/1km/config_init.xml b/testing_and_setup/compass/ocean/drying_slope/meshes/1km/config_init.xml index bdb361f464..bd6cc0ed3e 100644 --- a/testing_and_setup/compass/ocean/drying_slope/meshes/1km/config_init.xml +++ b/testing_and_setup/compass/ocean/drying_slope/meshes/1km/config_init.xml @@ -1,7 +1,7 @@ - + 6 28 1000.0 diff --git a/testing_and_setup/compass/ocean/drying_slope/meshes/250m/config_init.xml b/testing_and_setup/compass/ocean/drying_slope/meshes/250m/config_init.xml index 76015579be..5cebc44d04 100644 --- a/testing_and_setup/compass/ocean/drying_slope/meshes/250m/config_init.xml +++ b/testing_and_setup/compass/ocean/drying_slope/meshes/250m/config_init.xml @@ -1,7 +1,7 @@ - + 6 114 250.0 diff --git a/testing_and_setup/compass/ocean/internal_waves/5km/rpe_test/config_mpas_mesh.xml b/testing_and_setup/compass/ocean/internal_waves/5km/rpe_test/config_mpas_mesh.xml index d807f0a738..65622f63f3 100644 --- a/testing_and_setup/compass/ocean/internal_waves/5km/rpe_test/config_mpas_mesh.xml +++ b/testing_and_setup/compass/ocean/internal_waves/5km/rpe_test/config_mpas_mesh.xml @@ -1,7 +1,7 @@ - + 4 50 5000.0 diff --git a/testing_and_setup/compass/ocean/lock_exchange/0.5km/default/config_init1.xml b/testing_and_setup/compass/ocean/lock_exchange/0.5km/default/config_init1.xml index 1e38653a17..d2eb9c264b 100644 --- a/testing_and_setup/compass/ocean/lock_exchange/0.5km/default/config_init1.xml +++ b/testing_and_setup/compass/ocean/lock_exchange/0.5km/default/config_init1.xml @@ -47,7 +47,7 @@ v - + 6 30 1000.0 diff --git a/testing_and_setup/compass/ocean/lock_exchange/0.5km/rpe_test/config_mpas_mesh.xml b/testing_and_setup/compass/ocean/lock_exchange/0.5km/rpe_test/config_mpas_mesh.xml index 446d96165f..b624e50c71 100644 --- a/testing_and_setup/compass/ocean/lock_exchange/0.5km/rpe_test/config_mpas_mesh.xml +++ b/testing_and_setup/compass/ocean/lock_exchange/0.5km/rpe_test/config_mpas_mesh.xml @@ -1,7 +1,7 @@ - + 4 128 500.0 diff --git a/testing_and_setup/compass/ocean/overflow/1km/rpe_test/config_mpas_mesh.xml b/testing_and_setup/compass/ocean/overflow/1km/rpe_test/config_mpas_mesh.xml index 33e61b2978..e07c462330 100644 --- a/testing_and_setup/compass/ocean/overflow/1km/rpe_test/config_mpas_mesh.xml +++ b/testing_and_setup/compass/ocean/overflow/1km/rpe_test/config_mpas_mesh.xml @@ -1,7 +1,7 @@ - + 4 200 1000.0 diff --git a/testing_and_setup/compass/ocean/surface_waves/direct/1km/config_init1.xml b/testing_and_setup/compass/ocean/surface_waves/direct/1km/config_init1.xml index 8f1698ee6f..b39ac20129 100644 --- a/testing_and_setup/compass/ocean/surface_waves/direct/1km/config_init1.xml +++ b/testing_and_setup/compass/ocean/surface_waves/direct/1km/config_init1.xml @@ -54,7 +54,7 @@ - + 6 30 1000.0 diff --git a/testing_and_setup/compass/ocean/surface_waves/thickness_source/1km/config_init1.xml b/testing_and_setup/compass/ocean/surface_waves/thickness_source/1km/config_init1.xml index 8f1698ee6f..b39ac20129 100644 --- a/testing_and_setup/compass/ocean/surface_waves/thickness_source/1km/config_init1.xml +++ b/testing_and_setup/compass/ocean/surface_waves/thickness_source/1km/config_init1.xml @@ -54,7 +54,7 @@ - + 6 30 1000.0