From d3c149288ffdc22ad1bcc3f4f5ebb2aa299bcf55 Mon Sep 17 00:00:00 2001 From: Yue Ying Date: Fri, 27 Mar 2020 14:53:57 -0600 Subject: [PATCH 01/16] add obs type MPD_ABSOLUTE_HUMIDITY --- .../observations/DEFAULT_obs_kind_mod.F90 | 8 +- .../obs_def_abs_humidity_mod.f90 | 186 ++++++++++++++++++ 2 files changed, 193 insertions(+), 1 deletion(-) create mode 100644 observations/forward_operators/obs_def_abs_humidity_mod.f90 diff --git a/assimilation_code/modules/observations/DEFAULT_obs_kind_mod.F90 b/assimilation_code/modules/observations/DEFAULT_obs_kind_mod.F90 index a1c4e77451..24848aa66d 100644 --- a/assimilation_code/modules/observations/DEFAULT_obs_kind_mod.F90 +++ b/assimilation_code/modules/observations/DEFAULT_obs_kind_mod.F90 @@ -489,10 +489,14 @@ module obs_kind_mod QTY_ION_O_MIXING_RATIO = 365, & QTY_ATOMIC_H_MIXING_RATIO = 366 +!!MYY: absolute humidity +integer, parameter, public :: & + QTY_ABSOLUTE_HUMIDITY = 367 + ! max_defined_quantities is private to this module. see comment below near the max_obs_specific ! declaration for more info about publics and private values. -integer, parameter :: max_defined_quantities = 366 +integer, parameter :: max_defined_quantities = 367 !---------------------------------------------------------------------------- ! This list is autogenerated by the 'preprocess' program. To add new @@ -918,6 +922,8 @@ subroutine initialize_module obs_kind_names(365) = obs_kind_type(QTY_ION_O_MIXING_RATIO, 'QTY_ION_O_MIXING_RATIO') obs_kind_names(366) = obs_kind_type(QTY_ATOMIC_H_MIXING_RATIO, 'QTY_ATOMIC_H_MIXING_RATIO') +obs_kind_names(367) = obs_kind_type(QTY_ABSOLUTE_HUMIDITY, 'QTY_ABSOLUTE_HUMIDITY') + ! count here, then output below num_kind_assimilate = 0 diff --git a/observations/forward_operators/obs_def_abs_humidity_mod.f90 b/observations/forward_operators/obs_def_abs_humidity_mod.f90 new file mode 100644 index 0000000000..40b4a41278 --- /dev/null +++ b/observations/forward_operators/obs_def_abs_humidity_mod.f90 @@ -0,0 +1,186 @@ +! DART software - Copyright UCAR. This open source software is provided +! by UCAR, "as is", without charge, subject to all terms of use at +! http://www.image.ucar.edu/DAReS/DART/DART_download +! +! $Id$ + +! BEGIN DART PREPROCESS KIND LIST +!MPD_ABSOLUTE_HUMIDITY, QTY_ABSOLUTE_HUMIDITY +! END DART PREPROCESS KIND LIST + +! BEGIN DART PREPROCESS USE OF SPECIAL OBS_DEF MODULE +! use obs_def_abs_humidity_mod, only : get_expected_absolute_humidity +! END DART PREPROCESS USE OF SPECIAL OBS_DEF MODULE + +! BEGIN DART PREPROCESS GET_EXPECTED_OBS_FROM_DEF +! case(MPD_ABSOLUTE_HUMIDITY) +! call get_expected_absolute_humidity(state_handle, ens_size, location, expected_obs, istatus) +! END DART PREPROCESS GET_EXPECTED_OBS_FROM_DEF + +! BEGIN DART PREPROCESS READ_OBS_DEF +! case(MPD_ABSOLUTE_HUMIDITY) +! continue +! END DART PREPROCESS READ_OBS_DEF + +! BEGIN DART PREPROCESS WRITE_OBS_DEF +! case(MPD_ABSOLUTE_HUMIDITY) +! continue +! END DART PREPROCESS WRITE_OBS_DEF + +! BEGIN DART PREPROCESS INTERACTIVE_OBS_DEF +! case(MPD_ABSOLUTE_HUMIDITY) +! continue +! END DART PREPROCESS INTERACTIVE_OBS_DEF + +! BEGIN DART PREPROCESS MODULE CODE +module obs_def_abs_humidity_mod + +use types_mod, only : r8, missing_r8, gas_constant, gas_constant_v +use utilities_mod, only : register_module, error_handler, E_ERR, E_MSG, E_ALLMSG +use location_mod, only : location_type, set_location, get_location, write_location, & + read_location, is_vertical +use assim_model_mod, only : interpolate +use obs_kind_mod, only : QTY_TEMPERATURE, QTY_PRESSURE, QTY_VAPOR_MIXING_RATIO + +use ensemble_manager_mod, only : ensemble_type +use obs_def_utilities_mod, only : track_status + + +implicit none +private + +public :: get_expected_absolute_humidity + +! version controlled file description for error handling, do not edit +character(len=256), parameter :: source = & + "$URL$" +character(len=32 ), parameter :: revision = "$Revision$" +character(len=128), parameter :: revdate = "$Date$" + +logical, save :: module_initialized = .false. +logical, save :: first_time_warn_low = .true. +logical, save :: first_time_warn_high = .true. +character(len=512) :: msgstring +real(r8), parameter :: MIN_VALUE = 0.0 +real(r8), parameter :: MAX_VALUE = 0.1 + +contains + +!---------------------------------------------------------------------- + +subroutine initialize_module + +call register_module(source, revision, revdate) +module_initialized = .true. + +end subroutine initialize_module + +!---------------------------------------------------------------------------- + +subroutine get_expected_absolute_humidity(state_handle, ens_size, location, ah, istatus) + +type(ensemble_type), intent(in) :: state_handle +integer, intent(in) :: ens_size +type(location_type), intent(in) :: location +real(r8), intent(out) :: ah(ens_size) ! absolute humidity +integer, intent(out) :: istatus(ens_size) + +real(r8), dimension(ens_size) :: qvap, tmpk, pres, rho_dry, tmpv +integer, dimension(ens_size) :: qvap_istatus, tmpk_istatus, pres_istatus +real(r8) :: xyz(3) +logical :: return_now + +if ( .not. module_initialized ) call initialize_module + +istatus = 0 + +! interpolate the mixing ratio to the location +call interpolate(state_handle, ens_size, location, QTY_VAPOR_MIXING_RATIO, qvap, qvap_istatus) +call track_status(ens_size, qvap_istatus, ah, istatus, return_now) + +where (istatus == 0 .and. qvap < 0.0_r8) qvap = epsilon(0.0_r8) ! clamping? +where (istatus /= 0) istatus = 99 +if(return_now) return + +! interpolate the temperature to the desired location +call interpolate(state_handle, ens_size, location, QTY_TEMPERATURE, tmpk, tmpk_istatus) +call track_status(ens_size, tmpk_istatus, ah, istatus, return_now) + +where (tmpk <= 0.0_r8) + ah = missing_r8 + istatus = 99 +endwhere + +if(all(istatus /= 0)) return ! not using return_now because where clause modifies istatus. + +! interpolate the pressure, if observation location is not pressure +if ( is_vertical(location, "PRESSURE") ) then + xyz = get_location(location) + pres = xyz(3) +else + ! pressure comes back in pascals (not hPa or mb) + call interpolate(state_handle, ens_size, location, QTY_PRESSURE, pres, pres_istatus) + call track_status(ens_size, pres_istatus, ah, istatus, return_now) + + where (pres <= 0.0_r8 .or. pres >= 120000.0_r8) + ah = missing_r8 + istatus = 99 + end where + if(all(istatus /= 0)) return ! not using return_now because where clause modifies istatus. + +endif + +! Compute the ah +where (istatus == 0) + tmpv = tmpk * (1.0_r8 + qvap * gas_constant_v / gas_constant) + rho_dry = pres / (gas_constant * tmpv) + ah = qvap * rho_dry +end where + +! Warnings - first time only + +! nsc - my dilemma: each task does these obs independently. if you use E_ALLMSG you'll +! get a message from every processor which runs into this case. if you use E_MSG you'll +! only see those on task 0. for now i've removed the test and you'll get the message (once) +! even if there aren't any values outside the valid range. i can't see a good way to +! enforce that you get only one message when this could happen on any task and probably +! will happen on many tasks. + +if (first_time_warn_low) then + !if (any(ah < MIN_VALUE .and. istatus == 0)) then + ! write(msgstring, '(A,F12.6)') 'values lower than low limit detected, e.g.', minval(ah, istatus==0) + write(msgstring, *) 'checking absolute humidity value computed by the forward operator' + call error_handler(E_MSG,'get_expected_absolute_humidity', msgstring, & + text2='all values lower than 0 will be set to 0', & + text3='this message will only print once') + first_time_warn_low = .false. + !endif +endif +if (first_time_warn_high) then + !if (any(ah > MAX_VALUE .and. istatus == 0)) then + ! write(msgstring, '(A,F12.6)') 'values higher than high limit detected, e.g.', maxval(ah, istatus==0) + write(msgstring, *) 'checking absolute humidity value computed by the forward operator' + call error_handler(E_MSG,'get_expected_absolute_humidity', msgstring, & + text2='all values larger than 0.1 will be set to 0.1', & + text3='this message will only print once') + first_time_warn_high = .false. + !endif +endif + +! Actually force the absolute humidity to be between MIN_VALUE and MAX_VALUE +where(ah < MIN_VALUE .and. istatus == 0) ah = MIN_VALUE +where(ah > MAX_VALUE .and. istatus == 0) ah = MAX_VALUE + +return +end subroutine get_expected_absolute_humidity + +!---------------------------------------------------------------------------- + +end module obs_def_abs_humidity_mod +! END DART PREPROCESS MODULE CODE + +! +! $URL$ +! $Id$ +! $Revision$ +! $Date$ From a2d30439bd8d182ec08beb83a21040fe203a8d7b Mon Sep 17 00:00:00 2001 From: Yue Ying Date: Fri, 27 Mar 2020 15:05:52 -0600 Subject: [PATCH 02/16] obs_converter for MPD_ABSOLUTE_HUMIDITY --- .../obs_converters/MPD/text_to_obs.f90 | 206 ++++++++++++++++++ .../MPD/work/convert_to_text.py | 38 ++++ .../obs_converters/MPD/work/input.nml | 45 ++++ .../obs_converters/MPD/work/mkmf_preprocess | 18 ++ .../obs_converters/MPD/work/mkmf_text_to_obs | 18 ++ .../MPD/work/path_names_preprocess | 5 + .../MPD/work/path_names_text_to_obs | 27 +++ .../obs_converters/MPD/work/quickbuild.csh | 70 ++++++ 8 files changed, 427 insertions(+) create mode 100644 observations/obs_converters/MPD/text_to_obs.f90 create mode 100755 observations/obs_converters/MPD/work/convert_to_text.py create mode 100644 observations/obs_converters/MPD/work/input.nml create mode 100755 observations/obs_converters/MPD/work/mkmf_preprocess create mode 100755 observations/obs_converters/MPD/work/mkmf_text_to_obs create mode 100644 observations/obs_converters/MPD/work/path_names_preprocess create mode 100644 observations/obs_converters/MPD/work/path_names_text_to_obs create mode 100755 observations/obs_converters/MPD/work/quickbuild.csh diff --git a/observations/obs_converters/MPD/text_to_obs.f90 b/observations/obs_converters/MPD/text_to_obs.f90 new file mode 100644 index 0000000000..f94490a980 --- /dev/null +++ b/observations/obs_converters/MPD/text_to_obs.f90 @@ -0,0 +1,206 @@ +! DART software - Copyright UCAR. This open source software is provided +! by UCAR, "as is", without charge, subject to all terms of use at +! http://www.image.ucar.edu/DAReS/DART/DART_download +! +! DART $Id$ + +program text_to_obs + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +! +! text_to_obs - a program that only needs minor customization to read +! in a text-based dataset - either white-space separated values or +! fixed-width column data. +! +! created 29 Mar 2010 nancy collins NCAR/IMAGe +! modified 20 Mar 2020 Michael Ying +! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +use types_mod, only : r8, PI, DEG2RAD + +use utilities_mod, only : initialize_utilities, finalize_utilities, & + open_file, close_file + +use time_manager_mod, only : time_type, set_calendar_type, set_date, & + operator(>=), increment_time, get_time, & + operator(-), GREGORIAN, operator(+), print_date + +use location_mod, only : VERTISHEIGHT + +use obs_sequence_mod, only : obs_sequence_type, obs_type, read_obs_seq, & + static_init_obs_sequence, init_obs, write_obs_seq, & + init_obs_sequence, get_num_obs, & + set_copy_meta_data, set_qc_meta_data + +use obs_utilities_mod, only : create_3d_obs, add_obs_to_seq + +use obs_kind_mod, only : MPD_ABSOLUTE_HUMIDITY + +implicit none + +character(len=64), parameter :: text_input_file = 'text.txt' +character(len=64), parameter :: obs_out_file = 'obs_seq.out' + +logical, parameter :: debug = .false. ! set to .true. to print info + +character (len=129) :: input_line + +integer :: oday, osec, rcio, iunit, otype +integer :: year, month, day, hour, minute, second +integer :: num_copies, num_qc, max_obs + +logical :: first_obs + +real(r8) :: abs_humid, terr, qc +real(r8) :: lat, lon, vert + +type(obs_sequence_type) :: obs_seq +type(obs_type) :: obs, prev_obs +type(time_type) :: time_obs, prev_time + +! start of executable code + +call initialize_utilities('text_to_obs') + +! time setup +call set_calendar_type(GREGORIAN) + +! each observation in this series will have a single observation value +! and a quality control flag. the max possible number of obs needs to +! be specified but it will only write out the actual number created. +max_obs = 100000 +num_copies = 1 +num_qc = 1 + +! Set the DART data quality control. 0 is good data. +! increasingly larger QC values are more questionable quality data. +qc = 0.0_r8 + +! call the initialization code, and initialize two empty observation types +call static_init_obs_sequence() +call init_obs(obs, num_copies, num_qc) +call init_obs(prev_obs, num_copies, num_qc) +first_obs = .true. + +! create a new, empty obs_seq file. you must give a max limit +! on number of obs. increase the size if too small. +call init_obs_sequence(obs_seq, num_copies, num_qc, max_obs) + +! the first one needs to contain the string 'observation' and the +! second needs the string 'QC'. +call set_copy_meta_data(obs_seq, 1, 'observation') +call set_qc_meta_data(obs_seq, 1, 'Data QC') + +! if you want to append to existing files (e.g. you have a lot of +! small text files you want to combine), you can do it this way, +! or you can use the obs_sequence_tool to merge a list of files +! once they are in DART obs_seq format. + +! ! existing file found, append to it +! inquire(file=obs_out_file, exist=file_exist) +! if ( file_exist ) then +! call read_obs_seq(obs_out_file, 0, 0, max_obs, obs_seq) +! endif + +! open input text file +iunit = open_file(text_input_file, 'formatted', 'read') +if (debug) print *, 'opened input file ' // trim(text_input_file) + +obsloop: do ! no end limit - have the loop break when input ends + + ! read in a line from the text file. What you need to create an obs: + ! location: lat, lon, and height in pressure or meters + ! time: when the observation was taken + ! type: from the DART list of obs types + ! error: very important - the instrument error plus representativeness error + ! (see html file for more info) + + ! assume here a line is a type, location, time, value, obs error + + ! read in entire text line into a buffer + read(iunit, "(A)", iostat=rcio) input_line + if (rcio /= 0) then + if (debug) print *, 'got bad read code from input file, rcio = ', rcio + exit obsloop + endif + + ! pull off the first value as an integer, to decode the type + read(input_line, *, iostat=rcio) otype + if (rcio /= 0) then + if (debug) print *, 'got bad read code trying to get obs type, rcio = ', rcio + exit obsloop + endif + + if (debug) print *, 'next observation type = ', otype + + ! for this example, assume there is an obs type, where otype=1 is + ! abs humidity, the input text file has these as their own hardcoded convention. + + if (otype == 1) then !abs_humidity + read(input_line, *, iostat=rcio) otype, lat, lon, vert, & + year, month, day, hour, minute, second, & + abs_humid, terr + if (rcio /= 0) then + if (debug) print *, 'got bad read code getting rest of abs_humidity obs, rcio = ', rcio + exit obsloop + endif + + else !WHAT? + + ! no method defined to convert this type + write(*,*) "NOOOOO, DON'T DO THIS: no method defined to convert this type = ", otype + + endif + + if (debug) print *, 'next observation located at lat, lon = ', lat, lon + + ! check the lat/lon values to see if they are ok + if ( lat > 90.0_r8 .or. lat < -90.0_r8 ) cycle obsloop + if ( lon < 0.0_r8 .or. lon > 360.0_r8 ) cycle obsloop + + ! put date into a dart time format + time_obs = set_date(year, month, day, hour, minute, second) + + if (debug) call print_date(time_obs, 'next obs time is') + + ! extract time of observation into gregorian day, sec. + call get_time(time_obs, osec, oday) + + ! this example assumes there is an obs type, where otype=1 is + ! abs_humidity measured in height + if (otype == 1) then !abs_humidity + + ! height is in meters + ! make an obs derived type, and then add it to the sequence + call create_3d_obs(lat, lon, vert, VERTISHEIGHT, abs_humid, & + MPD_ABSOLUTE_HUMIDITY, terr, oday, osec, qc, obs) + call add_obs_to_seq(obs_seq, obs, time_obs, prev_obs, prev_time, first_obs) + if (debug) print *, 'added abs humidity obs to output seq' + + else !WHAT??? + + ! no method defined to convert this type + write(*,*) 'no method defined to convert this type = ', otype + + endif + +enddo obsloop + +! if we added any obs to the sequence, write it out to a file now. +if ( get_num_obs(obs_seq) > 0 ) then + if (debug) print *, 'writing obs_seq, obs_count = ', get_num_obs(obs_seq) + print *, 'writing obs_seq, obs_count = ', get_num_obs(obs_seq) + call write_obs_seq(obs_seq, obs_out_file) +endif + +! end of main program +call finalize_utilities() + +end program text_to_obs + +! +! $URL$ +! $Id$ +! $Revision$ +! $Date$ diff --git a/observations/obs_converters/MPD/work/convert_to_text.py b/observations/obs_converters/MPD/work/convert_to_text.py new file mode 100755 index 0000000000..251a8f5f52 --- /dev/null +++ b/observations/obs_converters/MPD/work/convert_to_text.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +import numpy as np +from netCDF4 import Dataset + +ccyy = 2019 ##date of data file +mm = 6 +dd = 14 +otype = 1 ##defined in obs_kind_mod, index for MPD_ABSOLUTE_HUMIDITY +n_sites = 5 ##number of MPD sites +obs_err = 0.001 ###set observation error 1g/m3 + +for site in range(1, n_sites+1): + f = Dataset('wv_mpd0{}.{:02d}{:02d}{:02d}.dafinal.Python.nc'.format(site, int(ccyy%100), mm, dd)) + dat = np.array(f.variables['Absolute_Humidity']) + dat = dat * 0.001 ##convert to kg/m3 + dat_mask = np.array(f.variables['Absolute_Humidity_mask']) + + nt, nz = dat.shape + tt = np.array(f.variables['time_Absolute_Humidity']) + zz = np.array(f.variables['range_Absolute_Humidity']) + + lat = np.array(f.variables['lidar_latitude']) + lon = np.array(f.variables['lidar_longitude']) + if(lon<0): + lon = 360.0 + lon + elev = np.array(f.variables['lidar_elevation']) + + for t in range(nt-1): + ii_sum = int(tt[t]/60) + hh = int(ii_sum/60) + ii = int(ii_sum%60) + for z in range(nz): + if(dat_mask[t,z]==0 and dat[t,z]>=0): ##QC mask and remove negative data + textfile = "work/{:04d}{:02d}{:02d}{:02d}{:02d}".format(ccyy, mm, dd, hh, ii) + f1 = open(textfile, "a") + string = "{:2d} {:9.5f} {:9.5f} {:8.1f} {:4d} {:02d} {:02d} {:02d} {:02d} {:02d} {:e} {:e}\n".format(otype, lat, lon, elev+zz[z], ccyy, mm, dd, hh, ii, 0, dat[t, z], obs_err) + f1.write(string) + f1.close() diff --git a/observations/obs_converters/MPD/work/input.nml b/observations/obs_converters/MPD/work/input.nml new file mode 100644 index 0000000000..0eb325ce73 --- /dev/null +++ b/observations/obs_converters/MPD/work/input.nml @@ -0,0 +1,45 @@ + +&preprocess_nml + overwrite_output = .true., + input_obs_kind_mod_file = '../../../../assimilation_code/modules/observations/DEFAULT_obs_kind_mod.F90', + output_obs_kind_mod_file = '../../../../assimilation_code/modules/observations/obs_kind_mod.f90', + input_obs_def_mod_file = '../../../../observations/forward_operators/DEFAULT_obs_def_mod.F90', + output_obs_def_mod_file = '../../../../observations/forward_operators/obs_def_mod.f90', + input_files = '../../../../observations/forward_operators/obs_def_abs_humidity_mod.f90', + / + +&obs_kind_nml + assimilate_these_obs_types = 'MPD_ABSOLUTE_HUMIDITY', + / + + +&location_nml + horiz_dist_only = .false., + vert_normalization_pressure = 100000.0, + vert_normalization_height = 100000.0, + vert_normalization_level = 20.0, + approximate_distance = .false., + nlon = 71, + nlat = 36, + output_box_info = .true. + / + +&utilities_nml + module_details = .false. + / + +&obs_sequence_nml + write_binary_obs_sequence = .false. + / + +&obs_sequence_tool_nml + filename_seq = 'obs_seq.out' + filename_seq_list = '' + filename_out = 'obs_seq.copy' + print_only = .false. + gregorian_cal = .true. + first_obs_days = -1 + first_obs_seconds = -1 + last_obs_days = -1 + last_obs_seconds = -1 + / diff --git a/observations/obs_converters/MPD/work/mkmf_preprocess b/observations/obs_converters/MPD/work/mkmf_preprocess new file mode 100755 index 0000000000..cb203268e4 --- /dev/null +++ b/observations/obs_converters/MPD/work/mkmf_preprocess @@ -0,0 +1,18 @@ +#!/bin/csh +# +# DART software - Copyright UCAR. This open source software is provided +# by UCAR, "as is", without charge, subject to all terms of use at +# http://www.image.ucar.edu/DAReS/DART/DART_download +# +# DART $Id$ + +../../../../build_templates/mkmf -p preprocess -t ../../../../build_templates/mkmf.template \ + -a "../../../.." path_names_preprocess + +exit $status + +# +# $URL$ +# $Revision$ +# $Date$ + diff --git a/observations/obs_converters/MPD/work/mkmf_text_to_obs b/observations/obs_converters/MPD/work/mkmf_text_to_obs new file mode 100755 index 0000000000..2049fd5043 --- /dev/null +++ b/observations/obs_converters/MPD/work/mkmf_text_to_obs @@ -0,0 +1,18 @@ +#!/bin/csh +# +# DART software - Copyright UCAR. This open source software is provided +# by UCAR, "as is", without charge, subject to all terms of use at +# http://www.image.ucar.edu/DAReS/DART/DART_download +# +# DART $Id$ + +../../../../build_templates/mkmf -p text_to_obs -t ../../../../build_templates/mkmf.template \ + -a "../../../.." path_names_text_to_obs + +exit $status + +# +# $URL$ +# $Revision$ +# $Date$ + diff --git a/observations/obs_converters/MPD/work/path_names_preprocess b/observations/obs_converters/MPD/work/path_names_preprocess new file mode 100644 index 0000000000..ae8022dafe --- /dev/null +++ b/observations/obs_converters/MPD/work/path_names_preprocess @@ -0,0 +1,5 @@ +assimilation_code/modules/utilities/null_mpi_utilities_mod.f90 +assimilation_code/modules/utilities/time_manager_mod.f90 +assimilation_code/modules/utilities/types_mod.f90 +assimilation_code/modules/utilities/utilities_mod.f90 +assimilation_code/programs/preprocess/preprocess.f90 diff --git a/observations/obs_converters/MPD/work/path_names_text_to_obs b/observations/obs_converters/MPD/work/path_names_text_to_obs new file mode 100644 index 0000000000..b551ddca46 --- /dev/null +++ b/observations/obs_converters/MPD/work/path_names_text_to_obs @@ -0,0 +1,27 @@ +assimilation_code/location/threed_sphere/location_mod.f90 +assimilation_code/modules/assimilation/adaptive_inflate_mod.f90 +assimilation_code/modules/assimilation/assim_model_mod.f90 +assimilation_code/modules/io/dart_time_io_mod.f90 +assimilation_code/modules/io/direct_netcdf_mod.f90 +assimilation_code/modules/io/io_filenames_mod.f90 +assimilation_code/modules/io/state_structure_mod.f90 +assimilation_code/modules/io/state_vector_io_mod.f90 +assimilation_code/modules/observations/obs_kind_mod.f90 +assimilation_code/modules/observations/obs_sequence_mod.f90 +assimilation_code/modules/utilities/distributed_state_mod.f90 +assimilation_code/modules/utilities/ensemble_manager_mod.f90 +assimilation_code/modules/utilities/netcdf_utilities_mod.f90 +assimilation_code/modules/utilities/null_mpi_utilities_mod.f90 +assimilation_code/modules/utilities/null_win_mod.f90 +assimilation_code/modules/utilities/options_mod.f90 +assimilation_code/modules/utilities/random_seq_mod.f90 +assimilation_code/modules/utilities/sort_mod.f90 +assimilation_code/modules/utilities/time_manager_mod.f90 +assimilation_code/modules/utilities/types_mod.f90 +assimilation_code/modules/utilities/utilities_mod.f90 +models/template/model_mod.f90 +models/utilities/default_model_mod.f90 +observations/forward_operators/obs_def_mod.f90 +observations/forward_operators/obs_def_utilities_mod.f90 +observations/obs_converters/MPD/text_to_obs.f90 +observations/obs_converters/utilities/obs_utilities_mod.f90 diff --git a/observations/obs_converters/MPD/work/quickbuild.csh b/observations/obs_converters/MPD/work/quickbuild.csh new file mode 100755 index 0000000000..a8513a91c4 --- /dev/null +++ b/observations/obs_converters/MPD/work/quickbuild.csh @@ -0,0 +1,70 @@ +#!/bin/csh +# +# DART software - Copyright UCAR. This open source software is provided +# by UCAR, "as is", without charge, subject to all terms of use at +# http://www.image.ucar.edu/DAReS/DART/DART_download +# +# DART $Id$ +# +# compile all converter programs + +#---------------------------------------------------------------------- +# 'preprocess' is a program that culls the appropriate sections of the +# observation module for the observations types in 'input.nml'; the +# resulting source file is used by all the remaining programs, +# so this MUST be run first. +#---------------------------------------------------------------------- + +set nonomatch +\rm -f preprocess *.o *.mod Makefile +\rm -f ../../../obs_def/obs_def_mod.f90 +\rm -f ../../../obs_kind/obs_kind_mod.f90 + +set MODEL = "MPD converters" + +@ n = 1 + +echo +echo +echo "---------------------------------------------------------------" +echo "${MODEL} build number ${n} is preprocess" + +csh mkmf_preprocess +make || exit $n + +./preprocess || exit 99 + +#---------------------------------------------------------------------- +# Build all the single-threaded targets +#---------------------------------------------------------------------- + +foreach TARGET ( mkmf_* ) + + set PROG = `echo $TARGET | sed -e 's#mkmf_##'` + + switch ( $TARGET ) + case mkmf_preprocess: + breaksw + default: + @ n = $n + 1 + echo + echo "---------------------------------------------------" + echo "${MODEL} build number ${n} is ${PROG}" + \rm -f ${PROG} + csh $TARGET || exit $n + make || exit $n + breaksw + endsw +end + +\rm -f *.o *.mod input.nml*_default Makefile .cppdefs + +echo "Success: All ${MODEL} programs compiled." + +exit 0 + +# +# $URL$ +# $Revision$ +# $Date$ + From 456d906da4810056578861717e84c78806e8f2a9 Mon Sep 17 00:00:00 2001 From: Yue Ying Date: Wed, 1 Apr 2020 17:05:31 -0600 Subject: [PATCH 03/16] remove version control tags --- observations/obs_converters/MPD/work/mkmf_preprocess | 7 ------- observations/obs_converters/MPD/work/mkmf_text_to_obs | 7 ------- observations/obs_converters/MPD/work/quickbuild.csh | 8 -------- 3 files changed, 22 deletions(-) diff --git a/observations/obs_converters/MPD/work/mkmf_preprocess b/observations/obs_converters/MPD/work/mkmf_preprocess index cb203268e4..1949f29ccd 100755 --- a/observations/obs_converters/MPD/work/mkmf_preprocess +++ b/observations/obs_converters/MPD/work/mkmf_preprocess @@ -3,16 +3,9 @@ # DART software - Copyright UCAR. This open source software is provided # by UCAR, "as is", without charge, subject to all terms of use at # http://www.image.ucar.edu/DAReS/DART/DART_download -# -# DART $Id$ ../../../../build_templates/mkmf -p preprocess -t ../../../../build_templates/mkmf.template \ -a "../../../.." path_names_preprocess exit $status -# -# $URL$ -# $Revision$ -# $Date$ - diff --git a/observations/obs_converters/MPD/work/mkmf_text_to_obs b/observations/obs_converters/MPD/work/mkmf_text_to_obs index 2049fd5043..1f14b27f94 100755 --- a/observations/obs_converters/MPD/work/mkmf_text_to_obs +++ b/observations/obs_converters/MPD/work/mkmf_text_to_obs @@ -3,16 +3,9 @@ # DART software - Copyright UCAR. This open source software is provided # by UCAR, "as is", without charge, subject to all terms of use at # http://www.image.ucar.edu/DAReS/DART/DART_download -# -# DART $Id$ ../../../../build_templates/mkmf -p text_to_obs -t ../../../../build_templates/mkmf.template \ -a "../../../.." path_names_text_to_obs exit $status -# -# $URL$ -# $Revision$ -# $Date$ - diff --git a/observations/obs_converters/MPD/work/quickbuild.csh b/observations/obs_converters/MPD/work/quickbuild.csh index a8513a91c4..96e38aaf38 100755 --- a/observations/obs_converters/MPD/work/quickbuild.csh +++ b/observations/obs_converters/MPD/work/quickbuild.csh @@ -4,8 +4,6 @@ # by UCAR, "as is", without charge, subject to all terms of use at # http://www.image.ucar.edu/DAReS/DART/DART_download # -# DART $Id$ -# # compile all converter programs #---------------------------------------------------------------------- @@ -62,9 +60,3 @@ end echo "Success: All ${MODEL} programs compiled." exit 0 - -# -# $URL$ -# $Revision$ -# $Date$ - From 95331d14bfdffccd36e1a1645ac49006d909adfe Mon Sep 17 00:00:00 2001 From: Yue Ying Date: Wed, 1 Apr 2020 17:05:56 -0600 Subject: [PATCH 04/16] add DART boilerplate --- observations/obs_converters/MPD/work/convert_to_text.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/observations/obs_converters/MPD/work/convert_to_text.py b/observations/obs_converters/MPD/work/convert_to_text.py index 251a8f5f52..5c938ad232 100755 --- a/observations/obs_converters/MPD/work/convert_to_text.py +++ b/observations/obs_converters/MPD/work/convert_to_text.py @@ -1,4 +1,8 @@ #!/usr/bin/env python3 +# +# DART software - Copyright UCAR. This open source software is provided +# by UCAR, "as is", without charge, subject to all terms of use at +# http://www.image.ucar.edu/DAReS/DART/DART_download import numpy as np from netCDF4 import Dataset From 73d722225f7ddbf4b78e1ea75a8a32c14175ee0f Mon Sep 17 00:00:00 2001 From: Yue Ying Date: Wed, 1 Apr 2020 17:06:38 -0600 Subject: [PATCH 05/16] minor changes of code to follow standard --- .../observations/DEFAULT_obs_kind_mod.F90 | 1 - .../obs_def_abs_humidity_mod.f90 | 38 +++++-------------- .../obs_converters/MPD/text_to_obs.f90 | 33 +++++----------- 3 files changed, 19 insertions(+), 53 deletions(-) diff --git a/assimilation_code/modules/observations/DEFAULT_obs_kind_mod.F90 b/assimilation_code/modules/observations/DEFAULT_obs_kind_mod.F90 index 24848aa66d..006375282a 100644 --- a/assimilation_code/modules/observations/DEFAULT_obs_kind_mod.F90 +++ b/assimilation_code/modules/observations/DEFAULT_obs_kind_mod.F90 @@ -489,7 +489,6 @@ module obs_kind_mod QTY_ION_O_MIXING_RATIO = 365, & QTY_ATOMIC_H_MIXING_RATIO = 366 -!!MYY: absolute humidity integer, parameter, public :: & QTY_ABSOLUTE_HUMIDITY = 367 diff --git a/observations/forward_operators/obs_def_abs_humidity_mod.f90 b/observations/forward_operators/obs_def_abs_humidity_mod.f90 index 40b4a41278..b8c9da220c 100644 --- a/observations/forward_operators/obs_def_abs_humidity_mod.f90 +++ b/observations/forward_operators/obs_def_abs_humidity_mod.f90 @@ -1,8 +1,6 @@ ! DART software - Copyright UCAR. This open source software is provided ! by UCAR, "as is", without charge, subject to all terms of use at ! http://www.image.ucar.edu/DAReS/DART/DART_download -! -! $Id$ ! BEGIN DART PREPROCESS KIND LIST !MPD_ABSOLUTE_HUMIDITY, QTY_ABSOLUTE_HUMIDITY @@ -52,10 +50,9 @@ module obs_def_abs_humidity_mod public :: get_expected_absolute_humidity ! version controlled file description for error handling, do not edit -character(len=256), parameter :: source = & - "$URL$" -character(len=32 ), parameter :: revision = "$Revision$" -character(len=128), parameter :: revdate = "$Date$" +character(len=*), parameter :: source = 'obs_def_abs_humidity_mod.f90' +character(len=*), parameter :: revision = '' +character(len=*), parameter :: revdate = '' logical, save :: module_initialized = .false. logical, save :: first_time_warn_low = .true. @@ -111,7 +108,7 @@ subroutine get_expected_absolute_humidity(state_handle, ens_size, location, ah, istatus = 99 endwhere -if(all(istatus /= 0)) return ! not using return_now because where clause modifies istatus. +if(all(istatus /= 0)) return ! interpolate the pressure, if observation location is not pressure if ( is_vertical(location, "PRESSURE") ) then @@ -126,8 +123,7 @@ subroutine get_expected_absolute_humidity(state_handle, ens_size, location, ah, ah = missing_r8 istatus = 99 end where - if(all(istatus /= 0)) return ! not using return_now because where clause modifies istatus. - + if(all(istatus /= 0)) return endif ! Compute the ah @@ -138,33 +134,23 @@ subroutine get_expected_absolute_humidity(state_handle, ens_size, location, ah, end where ! Warnings - first time only - -! nsc - my dilemma: each task does these obs independently. if you use E_ALLMSG you'll -! get a message from every processor which runs into this case. if you use E_MSG you'll -! only see those on task 0. for now i've removed the test and you'll get the message (once) -! even if there aren't any values outside the valid range. i can't see a good way to -! enforce that you get only one message when this could happen on any task and probably -! will happen on many tasks. - if (first_time_warn_low) then - !if (any(ah < MIN_VALUE .and. istatus == 0)) then - ! write(msgstring, '(A,F12.6)') 'values lower than low limit detected, e.g.', minval(ah, istatus==0) + if (any(ah < MIN_VALUE .and. istatus == 0)) then write(msgstring, *) 'checking absolute humidity value computed by the forward operator' call error_handler(E_MSG,'get_expected_absolute_humidity', msgstring, & text2='all values lower than 0 will be set to 0', & text3='this message will only print once') first_time_warn_low = .false. - !endif + endif endif if (first_time_warn_high) then - !if (any(ah > MAX_VALUE .and. istatus == 0)) then - ! write(msgstring, '(A,F12.6)') 'values higher than high limit detected, e.g.', maxval(ah, istatus==0) + if (any(ah > MAX_VALUE .and. istatus == 0)) then write(msgstring, *) 'checking absolute humidity value computed by the forward operator' call error_handler(E_MSG,'get_expected_absolute_humidity', msgstring, & text2='all values larger than 0.1 will be set to 0.1', & text3='this message will only print once') first_time_warn_high = .false. - !endif + endif endif ! Actually force the absolute humidity to be between MIN_VALUE and MAX_VALUE @@ -178,9 +164,3 @@ end subroutine get_expected_absolute_humidity end module obs_def_abs_humidity_mod ! END DART PREPROCESS MODULE CODE - -! -! $URL$ -! $Id$ -! $Revision$ -! $Date$ diff --git a/observations/obs_converters/MPD/text_to_obs.f90 b/observations/obs_converters/MPD/text_to_obs.f90 index f94490a980..0bc2c4e3f3 100644 --- a/observations/obs_converters/MPD/text_to_obs.f90 +++ b/observations/obs_converters/MPD/text_to_obs.f90 @@ -1,8 +1,6 @@ ! DART software - Copyright UCAR. This open source software is provided ! by UCAR, "as is", without charge, subject to all terms of use at ! http://www.image.ucar.edu/DAReS/DART/DART_download -! -! DART $Id$ program text_to_obs @@ -39,12 +37,12 @@ program text_to_obs implicit none -character(len=64), parameter :: text_input_file = 'text.txt' -character(len=64), parameter :: obs_out_file = 'obs_seq.out' +character(len=*), parameter :: text_input_file = 'text.txt' +character(len=*), parameter :: obs_out_file = 'obs_seq.out' logical, parameter :: debug = .false. ! set to .true. to print info -character (len=129) :: input_line +character (len=*) :: input_line integer :: oday, osec, rcio, iunit, otype integer :: year, month, day, hour, minute, second @@ -136,8 +134,7 @@ program text_to_obs ! for this example, assume there is an obs type, where otype=1 is ! abs humidity, the input text file has these as their own hardcoded convention. - - if (otype == 1) then !abs_humidity + if (otype == 1) then read(input_line, *, iostat=rcio) otype, lat, lon, vert, & year, month, day, hour, minute, second, & abs_humid, terr @@ -145,12 +142,9 @@ program text_to_obs if (debug) print *, 'got bad read code getting rest of abs_humidity obs, rcio = ', rcio exit obsloop endif - - else !WHAT? - - ! no method defined to convert this type - write(*,*) "NOOOOO, DON'T DO THIS: no method defined to convert this type = ", otype - + else + ! warning: no method defined to convert this type + write(*,*) "warning: no method defined to convert this type = ", otype endif if (debug) print *, 'next observation located at lat, lon = ', lat, lon @@ -169,7 +163,7 @@ program text_to_obs ! this example assumes there is an obs type, where otype=1 is ! abs_humidity measured in height - if (otype == 1) then !abs_humidity + if (otype == 1) then ! height is in meters ! make an obs derived type, and then add it to the sequence @@ -178,10 +172,9 @@ program text_to_obs call add_obs_to_seq(obs_seq, obs, time_obs, prev_obs, prev_time, first_obs) if (debug) print *, 'added abs humidity obs to output seq' - else !WHAT??? - + else ! no method defined to convert this type - write(*,*) 'no method defined to convert this type = ', otype + write(*,*) 'warning: no method defined to convert this type = ', otype endif @@ -198,9 +191,3 @@ program text_to_obs call finalize_utilities() end program text_to_obs - -! -! $URL$ -! $Id$ -! $Revision$ -! $Date$ From 91622c5b2ab1d8bfc13be6a3db64bddbfa1c8814 Mon Sep 17 00:00:00 2001 From: Yue Ying Date: Fri, 3 Apr 2020 10:10:16 -0600 Subject: [PATCH 06/16] the input_line needs len=129 --- observations/obs_converters/MPD/text_to_obs.f90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/observations/obs_converters/MPD/text_to_obs.f90 b/observations/obs_converters/MPD/text_to_obs.f90 index 0bc2c4e3f3..8cadc0a65c 100644 --- a/observations/obs_converters/MPD/text_to_obs.f90 +++ b/observations/obs_converters/MPD/text_to_obs.f90 @@ -42,7 +42,7 @@ program text_to_obs logical, parameter :: debug = .false. ! set to .true. to print info -character (len=*) :: input_line +character (len=129) :: input_line integer :: oday, osec, rcio, iunit, otype integer :: year, month, day, hour, minute, second From 5b8f643bc648665a235c722fa990c15f4c5c48f9 Mon Sep 17 00:00:00 2001 From: Tim Hoar Date: Mon, 13 Apr 2020 15:34:56 -0600 Subject: [PATCH 07/16] Adding converter from Michael Ying. This converter was based off the text_GITM converter which had some unneccesary logic. I removed that and improved the error handling. --- .../observations/DEFAULT_obs_kind_mod.F90 | 16 +-- .../obs_def_abs_humidity_mod.f90 | 24 ++-- .../obs_converters/MPD/text_to_obs.f90 | 106 +++++++++--------- .../MPD/work/convert_to_text.py | 4 +- .../obs_converters/MPD/work/input.nml | 34 ++---- .../obs_converters/MPD/work/mkmf_preprocess | 3 +- .../obs_converters/MPD/work/mkmf_text_to_obs | 3 +- .../obs_converters/MPD/work/quickbuild.csh | 2 - 8 files changed, 83 insertions(+), 109 deletions(-) diff --git a/assimilation_code/modules/observations/DEFAULT_obs_kind_mod.F90 b/assimilation_code/modules/observations/DEFAULT_obs_kind_mod.F90 index 006375282a..a6e8e6e432 100644 --- a/assimilation_code/modules/observations/DEFAULT_obs_kind_mod.F90 +++ b/assimilation_code/modules/observations/DEFAULT_obs_kind_mod.F90 @@ -1,8 +1,6 @@ ! DART software - Copyright UCAR. This open source software is provided ! by UCAR, "as is", without charge, subject to all terms of use at ! http://www.image.ucar.edu/DAReS/DART/DART_download -! -! $Id$ !---------------------------------------------------------------------- ! WARNING!! The file obs_kind_mod.f90 is AUTOGENERATED by the @@ -513,10 +511,9 @@ module obs_kind_mod !---------------------------------------------------------------------------- ! version controlled file description for error handling, do not edit -character(len=*), parameter :: source = & - "$URL$" -character(len=*), parameter :: revision = "$Revision$" -character(len=*), parameter :: revdate = "$Date$" +character(len=*), parameter :: source = 'DEFAULT_obs_def_mod.F90' +character(len=*), parameter :: revision = '' +character(len=*), parameter :: revdate = '' logical, save :: module_initialized = .false. @@ -921,7 +918,7 @@ subroutine initialize_module obs_kind_names(365) = obs_kind_type(QTY_ION_O_MIXING_RATIO, 'QTY_ION_O_MIXING_RATIO') obs_kind_names(366) = obs_kind_type(QTY_ATOMIC_H_MIXING_RATIO, 'QTY_ATOMIC_H_MIXING_RATIO') -obs_kind_names(367) = obs_kind_type(QTY_ABSOLUTE_HUMIDITY, 'QTY_ABSOLUTE_HUMIDITY') +obs_kind_names(367) = obs_kind_type(QTY_ABSOLUTE_HUMIDITY, 'QTY_ABSOLUTE_HUMIDITY') ! count here, then output below @@ -1492,8 +1489,3 @@ end function get_type_of_obs_from_menu end module obs_kind_mod -! -! $URL$ -! $Id$ -! $Revision$ -! $Date$ diff --git a/observations/forward_operators/obs_def_abs_humidity_mod.f90 b/observations/forward_operators/obs_def_abs_humidity_mod.f90 index b8c9da220c..02713371b4 100644 --- a/observations/forward_operators/obs_def_abs_humidity_mod.f90 +++ b/observations/forward_operators/obs_def_abs_humidity_mod.f90 @@ -3,7 +3,7 @@ ! http://www.image.ucar.edu/DAReS/DART/DART_download ! BEGIN DART PREPROCESS KIND LIST -!MPD_ABSOLUTE_HUMIDITY, QTY_ABSOLUTE_HUMIDITY +! MPD_ABSOLUTE_HUMIDITY, QTY_ABSOLUTE_HUMIDITY ! END DART PREPROCESS KIND LIST ! BEGIN DART PREPROCESS USE OF SPECIAL OBS_DEF MODULE @@ -31,8 +31,11 @@ ! END DART PREPROCESS INTERACTIVE_OBS_DEF ! BEGIN DART PREPROCESS MODULE CODE + module obs_def_abs_humidity_mod +! Module contributed by Michael Ying. Thank You Michael! + use types_mod, only : r8, missing_r8, gas_constant, gas_constant_v use utilities_mod, only : register_module, error_handler, E_ERR, E_MSG, E_ALLMSG use location_mod, only : location_type, set_location, get_location, write_location, & @@ -43,7 +46,6 @@ module obs_def_abs_humidity_mod use ensemble_manager_mod, only : ensemble_type use obs_def_utilities_mod, only : track_status - implicit none private @@ -57,7 +59,7 @@ module obs_def_abs_humidity_mod logical, save :: module_initialized = .false. logical, save :: first_time_warn_low = .true. logical, save :: first_time_warn_high = .true. -character(len=512) :: msgstring +character(len=512) :: string1, string2 real(r8), parameter :: MIN_VALUE = 0.0 real(r8), parameter :: MAX_VALUE = 0.1 @@ -136,19 +138,19 @@ subroutine get_expected_absolute_humidity(state_handle, ens_size, location, ah, ! Warnings - first time only if (first_time_warn_low) then if (any(ah < MIN_VALUE .and. istatus == 0)) then - write(msgstring, *) 'checking absolute humidity value computed by the forward operator' - call error_handler(E_MSG,'get_expected_absolute_humidity', msgstring, & - text2='all values lower than 0 will be set to 0', & - text3='this message will only print once') + write(string1, *) 'checking absolute humidity value computed by the forward operator' + write(string2,*)'all values lower than ',MIN_VALUE,' will be set to ',MIN_VALUE + call error_handler(E_MSG,'get_expected_absolute_humidity', string1, & + text2=string2, text3='this message will only print once') first_time_warn_low = .false. endif endif if (first_time_warn_high) then if (any(ah > MAX_VALUE .and. istatus == 0)) then - write(msgstring, *) 'checking absolute humidity value computed by the forward operator' - call error_handler(E_MSG,'get_expected_absolute_humidity', msgstring, & - text2='all values larger than 0.1 will be set to 0.1', & - text3='this message will only print once') + write(string1, *) 'checking absolute humidity value computed by the forward operator' + write(string2,*)'all values larger than ',MAX_VALUE,' will be set to ',MAX_VALUE + call error_handler(E_MSG,'get_expected_absolute_humidity', string1, & + text2=string2, text3='this message will only print once') first_time_warn_high = .false. endif endif diff --git a/observations/obs_converters/MPD/text_to_obs.f90 b/observations/obs_converters/MPD/text_to_obs.f90 index 8cadc0a65c..e28e9e26fb 100644 --- a/observations/obs_converters/MPD/text_to_obs.f90 +++ b/observations/obs_converters/MPD/text_to_obs.f90 @@ -6,22 +6,22 @@ program text_to_obs !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ! -! text_to_obs - a program that only needs minor customization to read +! MPD/text_to_obs - a program that only needs minor customization to read ! in a text-based dataset - either white-space separated values or ! fixed-width column data. ! -! created 29 Mar 2010 nancy collins NCAR/IMAGe -! modified 20 Mar 2020 Michael Ying +! created 20 Mar 2020 Michael Ying ! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! use types_mod, only : r8, PI, DEG2RAD use utilities_mod, only : initialize_utilities, finalize_utilities, & - open_file, close_file + open_file, close_file, error_handler, & + E_ERR, E_WARN, E_MSG use time_manager_mod, only : time_type, set_calendar_type, set_date, & - operator(>=), increment_time, get_time, & + operator(>=), get_time, & operator(-), GREGORIAN, operator(+), print_date use location_mod, only : VERTISHEIGHT @@ -33,18 +33,20 @@ program text_to_obs use obs_utilities_mod, only : create_3d_obs, add_obs_to_seq -use obs_kind_mod, only : MPD_ABSOLUTE_HUMIDITY +use obs_kind_mod, only : get_num_quantities, MPD_ABSOLUTE_HUMIDITY implicit none +character(len=*), parameter :: source = 'MPD/text_to_obs.f90' character(len=*), parameter :: text_input_file = 'text.txt' character(len=*), parameter :: obs_out_file = 'obs_seq.out' logical, parameter :: debug = .false. ! set to .true. to print info -character (len=129) :: input_line +character(len=129) :: input_line +character(len=512) :: string1 -integer :: oday, osec, rcio, iunit, otype +integer :: oday, osec, rcio, iunit, ilinecount integer :: year, month, day, hour, minute, second integer :: num_copies, num_qc, max_obs @@ -59,9 +61,8 @@ program text_to_obs ! start of executable code -call initialize_utilities('text_to_obs') +call initialize_utilities(source) -! time setup call set_calendar_type(GREGORIAN) ! each observation in this series will have a single observation value @@ -90,6 +91,14 @@ program text_to_obs call set_copy_meta_data(obs_seq, 1, 'observation') call set_qc_meta_data(obs_seq, 1, 'Data QC') +! This is sort of a do-nothing call to initialize the obs_kind_mod module. +! The only real purpose of this call is to make obs_kind_mod print its +! initialization report at the beginning of the output from text_to_obs. +! Without this call, the initialization report actually comes AFTER all +! the observations have been processed and right before the output file +! is closed. We do not actually need to know what is in string1. +write(string1,*)' num_observations understood is ',get_num_quantities() + ! if you want to append to existing files (e.g. you have a lot of ! small text files you want to combine), you can do it this way, ! or you can use the obs_sequence_tool to merge a list of files @@ -105,6 +114,8 @@ program text_to_obs iunit = open_file(text_input_file, 'formatted', 'read') if (debug) print *, 'opened input file ' // trim(text_input_file) +ilinecount = 1 + obsloop: do ! no end limit - have the loop break when input ends ! read in a line from the text file. What you need to create an obs: @@ -114,42 +125,37 @@ program text_to_obs ! error: very important - the instrument error plus representativeness error ! (see html file for more info) - ! assume here a line is a type, location, time, value, obs error - ! read in entire text line into a buffer read(iunit, "(A)", iostat=rcio) input_line - if (rcio /= 0) then - if (debug) print *, 'got bad read code from input file, rcio = ', rcio + if (rcio == 0) then + continue ! line read normally + elseif (rcio < 0) then ! Normal end-of-file exit obsloop + else + write(string1,*)'got bad read code from input file, rcio = ', rcio, & + ', line number =',ilinecount + call error_handler(E_ERR,source,string1,text2='Hard Stop. No output.') endif - ! pull off the first value as an integer, to decode the type - read(input_line, *, iostat=rcio) otype + ! the input text file has its own hardcoded convention for abs humidity + ! 'lat' is degrees Latitude [-90,90] + ! 'lon' is degrees East Longitude [0,360] + ! 'vert' is height in meters + ! 'terr' is the observation error STANDARD DEVIATION, this is converted into + ! units of variance inside DART. This should include both the instrument + ! error and an estimate of the representativeness error. + + read(input_line, *, iostat=rcio) lat, lon, vert, & + year, month, day, hour, minute, second, & + abs_humid, terr if (rcio /= 0) then - if (debug) print *, 'got bad read code trying to get obs type, rcio = ', rcio - exit obsloop + write(string1,*)'Unable to parse line ',ilinecount,', rcio = ', rcio + call error_handler(E_ERR,source,string1) endif - if (debug) print *, 'next observation type = ', otype - - ! for this example, assume there is an obs type, where otype=1 is - ! abs humidity, the input text file has these as their own hardcoded convention. - if (otype == 1) then - read(input_line, *, iostat=rcio) otype, lat, lon, vert, & - year, month, day, hour, minute, second, & - abs_humid, terr - if (rcio /= 0) then - if (debug) print *, 'got bad read code getting rest of abs_humidity obs, rcio = ', rcio - exit obsloop - endif - else - ! warning: no method defined to convert this type - write(*,*) "warning: no method defined to convert this type = ", otype - endif - - if (debug) print *, 'next observation located at lat, lon = ', lat, lon + if (debug) print *, 'observation ',ilinecount,' located at lat, lon = ', lat, lon - ! check the lat/lon values to see if they are ok + ! skip any observations outside these bounds if ( lat > 90.0_r8 .or. lat < -90.0_r8 ) cycle obsloop if ( lon < 0.0_r8 .or. lon > 360.0_r8 ) cycle obsloop @@ -161,29 +167,23 @@ program text_to_obs ! extract time of observation into gregorian day, sec. call get_time(time_obs, osec, oday) - ! this example assumes there is an obs type, where otype=1 is - ! abs_humidity measured in height - if (otype == 1) then - - ! height is in meters - ! make an obs derived type, and then add it to the sequence - call create_3d_obs(lat, lon, vert, VERTISHEIGHT, abs_humid, & - MPD_ABSOLUTE_HUMIDITY, terr, oday, osec, qc, obs) - call add_obs_to_seq(obs_seq, obs, time_obs, prev_obs, prev_time, first_obs) - if (debug) print *, 'added abs humidity obs to output seq' + ! make an obs derived type, and then add it to the sequence + call create_3d_obs(lat, lon, vert, VERTISHEIGHT, abs_humid, & + MPD_ABSOLUTE_HUMIDITY, terr, oday, osec, qc, obs) + call add_obs_to_seq(obs_seq, obs, time_obs, prev_obs, prev_time, first_obs) - else - ! no method defined to convert this type - write(*,*) 'warning: no method defined to convert this type = ', otype + if (debug) print *, 'added abs humidity obs to output seq' - endif + ilinecount = ilinecount + 1 enddo obsloop +call close_file(iunit) + ! if we added any obs to the sequence, write it out to a file now. if ( get_num_obs(obs_seq) > 0 ) then - if (debug) print *, 'writing obs_seq, obs_count = ', get_num_obs(obs_seq) - print *, 'writing obs_seq, obs_count = ', get_num_obs(obs_seq) + write(string1,*)'writing "'//trim(obs_out_file)//'", obs_count = ', get_num_obs(obs_seq) + call error_handler(E_MSG,source,string1) call write_obs_seq(obs_seq, obs_out_file) endif diff --git a/observations/obs_converters/MPD/work/convert_to_text.py b/observations/obs_converters/MPD/work/convert_to_text.py index 5c938ad232..f241b2d2d3 100755 --- a/observations/obs_converters/MPD/work/convert_to_text.py +++ b/observations/obs_converters/MPD/work/convert_to_text.py @@ -3,13 +3,13 @@ # DART software - Copyright UCAR. This open source software is provided # by UCAR, "as is", without charge, subject to all terms of use at # http://www.image.ucar.edu/DAReS/DART/DART_download + import numpy as np from netCDF4 import Dataset ccyy = 2019 ##date of data file mm = 6 dd = 14 -otype = 1 ##defined in obs_kind_mod, index for MPD_ABSOLUTE_HUMIDITY n_sites = 5 ##number of MPD sites obs_err = 0.001 ###set observation error 1g/m3 @@ -37,6 +37,6 @@ if(dat_mask[t,z]==0 and dat[t,z]>=0): ##QC mask and remove negative data textfile = "work/{:04d}{:02d}{:02d}{:02d}{:02d}".format(ccyy, mm, dd, hh, ii) f1 = open(textfile, "a") - string = "{:2d} {:9.5f} {:9.5f} {:8.1f} {:4d} {:02d} {:02d} {:02d} {:02d} {:02d} {:e} {:e}\n".format(otype, lat, lon, elev+zz[z], ccyy, mm, dd, hh, ii, 0, dat[t, z], obs_err) + string = "{:9.5f} {:9.5f} {:8.1f} {:4d} {:02d} {:02d} {:02d} {:02d} {:02d} {:e} {:e}\n".format(lat, lon, elev+zz[z], ccyy, mm, dd, hh, ii, 0, dat[t, z], obs_err) f1.write(string) f1.close() diff --git a/observations/obs_converters/MPD/work/input.nml b/observations/obs_converters/MPD/work/input.nml index 0eb325ce73..dfe2a8e614 100644 --- a/observations/obs_converters/MPD/work/input.nml +++ b/observations/obs_converters/MPD/work/input.nml @@ -1,27 +1,18 @@ &preprocess_nml - overwrite_output = .true., - input_obs_kind_mod_file = '../../../../assimilation_code/modules/observations/DEFAULT_obs_kind_mod.F90', - output_obs_kind_mod_file = '../../../../assimilation_code/modules/observations/obs_kind_mod.f90', - input_obs_def_mod_file = '../../../../observations/forward_operators/DEFAULT_obs_def_mod.F90', - output_obs_def_mod_file = '../../../../observations/forward_operators/obs_def_mod.f90', - input_files = '../../../../observations/forward_operators/obs_def_abs_humidity_mod.f90', + overwrite_output = .true. + input_obs_kind_mod_file = '../../../../assimilation_code/modules/observations/DEFAULT_obs_kind_mod.F90' + output_obs_kind_mod_file = '../../../../assimilation_code/modules/observations/obs_kind_mod.f90' + input_obs_def_mod_file = '../../../../observations/forward_operators/DEFAULT_obs_def_mod.F90' + output_obs_def_mod_file = '../../../../observations/forward_operators/obs_def_mod.f90' + input_files = '../../../../observations/forward_operators/obs_def_abs_humidity_mod.f90' / &obs_kind_nml - assimilate_these_obs_types = 'MPD_ABSOLUTE_HUMIDITY', + assimilate_these_obs_types = 'MPD_ABSOLUTE_HUMIDITY' / - &location_nml - horiz_dist_only = .false., - vert_normalization_pressure = 100000.0, - vert_normalization_height = 100000.0, - vert_normalization_level = 20.0, - approximate_distance = .false., - nlon = 71, - nlat = 36, - output_box_info = .true. / &utilities_nml @@ -32,14 +23,3 @@ write_binary_obs_sequence = .false. / -&obs_sequence_tool_nml - filename_seq = 'obs_seq.out' - filename_seq_list = '' - filename_out = 'obs_seq.copy' - print_only = .false. - gregorian_cal = .true. - first_obs_days = -1 - first_obs_seconds = -1 - last_obs_days = -1 - last_obs_seconds = -1 - / diff --git a/observations/obs_converters/MPD/work/mkmf_preprocess b/observations/obs_converters/MPD/work/mkmf_preprocess index 1949f29ccd..1ac1f154a8 100755 --- a/observations/obs_converters/MPD/work/mkmf_preprocess +++ b/observations/obs_converters/MPD/work/mkmf_preprocess @@ -4,7 +4,8 @@ # by UCAR, "as is", without charge, subject to all terms of use at # http://www.image.ucar.edu/DAReS/DART/DART_download -../../../../build_templates/mkmf -p preprocess -t ../../../../build_templates/mkmf.template \ + ../../../../build_templates/mkmf -p preprocess \ + -t ../../../../build_templates/mkmf.template \ -a "../../../.." path_names_preprocess exit $status diff --git a/observations/obs_converters/MPD/work/mkmf_text_to_obs b/observations/obs_converters/MPD/work/mkmf_text_to_obs index 1f14b27f94..c9a748ebff 100755 --- a/observations/obs_converters/MPD/work/mkmf_text_to_obs +++ b/observations/obs_converters/MPD/work/mkmf_text_to_obs @@ -4,7 +4,8 @@ # by UCAR, "as is", without charge, subject to all terms of use at # http://www.image.ucar.edu/DAReS/DART/DART_download -../../../../build_templates/mkmf -p text_to_obs -t ../../../../build_templates/mkmf.template \ + ../../../../build_templates/mkmf -p text_to_obs \ + -t ../../../../build_templates/mkmf.template \ -a "../../../.." path_names_text_to_obs exit $status diff --git a/observations/obs_converters/MPD/work/quickbuild.csh b/observations/obs_converters/MPD/work/quickbuild.csh index 96e38aaf38..e4fdb08018 100755 --- a/observations/obs_converters/MPD/work/quickbuild.csh +++ b/observations/obs_converters/MPD/work/quickbuild.csh @@ -3,8 +3,6 @@ # DART software - Copyright UCAR. This open source software is provided # by UCAR, "as is", without charge, subject to all terms of use at # http://www.image.ucar.edu/DAReS/DART/DART_download -# -# compile all converter programs #---------------------------------------------------------------------- # 'preprocess' is a program that culls the appropriate sections of the From 1451988e59c69973ac381cd3b3fd4fd233fc05e2 Mon Sep 17 00:00:00 2001 From: Tim Hoar Date: Wed, 15 Apr 2020 20:32:48 -0600 Subject: [PATCH 08/16] Adding absolute humidity observation converter necessitated changing the observations.html document, so I converted it to README.md so it would be more 'viewable' on GitHub. necessitated adding something to the CHANGELOG, so I converted it to CHANGELOG.md in the hopes it will be 'prettier'. No content changes, but I did notice that the support for DART QC 8 got added on several occasions. not great. --- CHANGELOG | 913 ------------------ CHANGELOG.md | 885 +++++++++++++++++ .../observations/obs_sequence_mod.html | 2 +- .../create_obs_sequence.html | 2 +- docs/pages/Models.md | 2 +- docs/pages/Observations.md | 4 +- .../Ameriflux/level4_to_obs.html | 2 +- .../obs_converters/MODIS/MOD15A2_to_obs.html | 2 +- .../{text_to_obs.f90 => MPD_text_to_obs.f90} | 10 +- .../obs_converters/MPD/work/.gitignore | 1 + ...{mkmf_text_to_obs => mkmf_MPD_text_to_obs} | 4 +- ...text_to_obs => path_names_MPD_text_to_obs} | 2 +- observations/obs_converters/README | 22 - observations/obs_converters/README.md | 393 ++++++++ observations/obs_converters/SST/SST.html | 2 +- .../obs_converters/text/text_to_obs.html | 2 +- 16 files changed, 1296 insertions(+), 952 deletions(-) delete mode 100644 CHANGELOG create mode 100644 CHANGELOG.md rename observations/obs_converters/MPD/{text_to_obs.f90 => MPD_text_to_obs.f90} (96%) create mode 100644 observations/obs_converters/MPD/work/.gitignore rename observations/obs_converters/MPD/work/{mkmf_text_to_obs => mkmf_MPD_text_to_obs} (72%) rename observations/obs_converters/MPD/work/{path_names_text_to_obs => path_names_MPD_text_to_obs} (96%) delete mode 100644 observations/obs_converters/README create mode 100644 observations/obs_converters/README.md diff --git a/CHANGELOG b/CHANGELOG deleted file mode 100644 index fd9b32a8a1..0000000000 --- a/CHANGELOG +++ /dev/null @@ -1,913 +0,0 @@ -# DART software - Copyright UCAR. This open source software is provided -# by UCAR, "as is", without charge, subject to all terms of use at -# http://www.image.ucar.edu/DAReS/DART/DART_download -# -# DART $Id$ - -SEE THE BOTTOM OF THIS FILE FOR THE MOST RECENT CHANGES! - -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -+ Jan 13th 2017 :: rma_fixed_filenames merge changes. Revision: 10902 -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -Specific namelist changes include: - -1. ) Earlier versions of the RMA branch code supported both direct NetCDF - reads/writes and the original binary/ascii DART format restart files. - As of the next update DART format files are no longer supported. All - I/O is NetCDF only. If your model does not use NetCDF you will still - need a model_to_dart and dart_to_model converter; otherwise all DART - programs read the model's NetCDF files directly. The namelist options - related to selecting direct netcdf I/O have been removed. - -2. ) Diagnostic and state space data (such as inflation, mean and sd - information) that were previously stored in {Prior,Posterior}_Diag.nc - are now broken up into multiple files and have fixed filenames. This - decreases the IO time for diagnostic output and reduces the number of - namelist options. - -3. ) There is no longer support for observation space inflation - (i.e. inf_flavor = 1). Contact us at dart@ucar.edu if you have an - interest in using this option. - ------------------------------------------------------------------------------- -Changes to the filter_nml are : ------------------------------------------------------------------------------- -* restart_in_file_name -- has been replaced with input_restart_file_list. - The namelist must contain one or more file names, - each of which is a textfile containing a list of N - NetCDF restart files, one per line for each ensemble member. - For models with multiple domains (e.g. nested WRF or - CLM) you must specify a listfile for each domain. - -* restart_out_file_name -- has been replaced with output_restart_file_list. - Same format as input_restart_file_list. - -* inf_in_file_name -- REMOVED, now have fixed names of the form - input_{prior,posterior}inf_{mean,sd}.nc - -* inf_out_file_name -- REMOVED, now have fixed names of the form - output_{prior,posterior}inf_{mean,sd}.nc. - -* inf_diag_filename -- REMOVED - -* inf_output_restart -- REMOVED, inflation restarts will be written - out if inflation is turned on - -* output_inflation -- REMOVED, inflation diagnostic files will be written - if inflation is turned on - -* stages_to_write -- There is more control over what state data - to write. Options are at stages : - 'input', 'preassim', postassim', 'output'. - Stages preassim and postassim will output - state data originally contained within the - copies of Prior_Diag.nc and Posterior_Diag.nc. - See rma_doc/rma.html for details on the - filename conventions. For example, running - filter with prior inflation enabled with - stage 'preassim' enabled will produce files - with names: - preassim_member_####.nc - preassim_{mean,sd}.nc - preassim_priorinf_{mean,sd}.nc - -* write_all_stages_at_end -- important for large models - all output file - I/O is deferred until the end of filter, but - will use more memory to store the data. More - detailed info is in rma_doc/rma.html - -* output_restart_mean -- renamed output_mean - -* output_restart -- renamed output_restarts - -* direct_netcdf_{read,write} -- REMOVED, always true - -* restart_list_file -- renamed input_restart_file_list - -* single_restart_file_in -- renamed single_file_in - -* single_restart_file_out -- renamed single_file_out - -* add_domain_extension -- REMOVED - -* use_restart_list -- REMOVED - -* overwrite_state_input -- REMOVED, equivalent functionality can be set - with single_restart_file_in = single_restart_file_out - ------------------------------------------------------------------------------- -Changes to the perfect_model_obs_nml are : ------------------------------------------------------------------------------- -* restart_in_filename -- renamed restart_in_file_names takes a NetCDF - file. For multiple domains you can specify a - list. - -* direct_netcdf_{read,write} -- REMOVED, always true - ------------------------------------------------------------------------------- -Changes to the state_space_diag_nml are : ------------------------------------------------------------------------------- -* single_file -- REMOVED, diagnostic files are now controlled - in filter_nml with stages_to_write - -* make_diagnostic_files -- REMOVED, no longer produce original - Prior_Diag.nc and Posterior_Diag.nc - -* netCDF_large_file_support -- REMOVED, always true - ------------------------------------------------------------------------------- -Changes to the state_vector_io_nml are : ------------------------------------------------------------------------------- -* write_binary_restart_files -- REMOVED - ------------------------------------------------------------------------------- -Changes to the ensemble_manager_nml are : ------------------------------------------------------------------------------- -* flag_unneeded_transposes -- REMOVED - ------------------------------------------------------------------------------- -Changes to the integrate_model_nml are : ------------------------------------------------------------------------------- -* advance_restart_format -- REMOVED, only supporting NetCDF format. - ------------------------------------------------------------------------------- -Scripting with CESM ------------------------------------------------------------------------------- -See models/cam-fv/scripts_cesm1_5/assimilate.csh for an example of how to -handle the new filename conventions. - -(To help find things: input_priorinf_mean output_priorinf_mean ) -{in,out}put_{prior,post}inf_{mean,sd}.nc ARE in use; - Search for stage_metadata%filenames turned up - interface set_file_metadata - module procedure set_explicit_file_metadata - module procedure set_stage_file_metadata - - ! stage_name is {input,preassim,postassim,output} - ! base_name is {mean,sd,{prior,post}inf_{mean,sd}} from filter/filter_mod.f90. - write(string1,'(A,''.nc'')') trim(stage_name)//'_'//trim(base_name) - file_info%stage_metadata%filenames(my_copy,1) = trim(string1) - - This shows where inflation file names are defined. - > grep -I set_file_metadata */*.f90 | grep inf - filter/filter_mod.f90: - call set_file_metadata(file_info, PRIOR_INF_MEAN, stage, 'priorinf_mean', 'prior inflation mean') - call set_file_metadata(file_info, PRIOR_INF_SD, stage, 'priorinf_sd', 'prior inflation sd') - call set_file_metadata(file_info, POST_INF_MEAN, stage, 'postinf_mean', 'posterior inflation mean') - call set_file_metadata(file_info, POST_INF_SD, stage, 'postinf_sd', 'posterior inflation sd') - - subroutine set_member_file_metadata(file_info, ens_size, my_copy_start) - call set_file_metadata(file_info, icopy, stage_name, base_name, desc, offset) - - subroutine set_stage_file_metadata(file_info, copy_number, stage, base_name, desc, offset) - write(string1,'(A,''.nc'')') trim(stage_name)//'_'//trim(base_name) - - subroutine set_explicit_file_metadata(file_info, cnum, fnames, desc) - file_info%stage_metadata%filenames(cnum,idom) = trim(fnames(idom)) - file_info%stage_metadata%file_description(cnum,idom) = trim(string1) - - function construct_file_names(file_info, ens_size, copy, domain) - write(construct_file_names, '(A, ''_member_'', I4.4, A, ''.nc'')') & - trim(file_info%root_name), copy, trim(dom_str) - - - -Also see - harnesses/filename_harness/files: ENS_MEAN_COPY PriorDiag_mean.nc - - ------------------------------------------------------------------------------- -ADDITIONAL NOTES : ------------------------------------------------------------------------------- -* currently the closest_member_tool is broken but plans on being fixed soon. -* restart_file_tool and most model_to_dart/dart_to_model programs have been - deprecated, since DART formated restarts are no longer supported. -* some programs such as model_mod_check have not been fully tested and need - to be exercised with the new naming conventions. - - - -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -+ Feb 15th 2017 :: rma_single_file merge changes. Revision: 11136 -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -Filter and PMO can now run with multiple cycles for low order models. The output -for this is only supported with single file output (members, inflation, mean, sd -are all in the same file). - -Added matlab support for diagnostics format in lower order models. - ------------------------------------------------------------------------------- -Changes to the filter_nml are : ------------------------------------------------------------------------------- - -output_restart -- RENAMED to output_members -restart_in_file_name -- RENAMED to input_state_file_list -restart_out_file_name -- RENAMED to output_state_file_list -single_restart_file_in -- RENAMED to single_file_in -single_restart_file_out -- RENAMED to single_file_out - -input_state_files -- ADDED - not currently working -output_state_files -- ADDED - not currently working - -has_cycling -- ADDED for low order models - ------------------------------------------------------------------------------- -Changes to the perfect_model_obs_nml are : ------------------------------------------------------------------------------- - -start_from_restart -- RENAMED read_input_state_from_file -output_restart -- RENAMED write_output_state_to_file -restart_in_file_name -- RENAMED input_state_files -restart_out_file_name -- RENAMED output_state_files - -single_file_in -- ADDED for low order models -single_file_out -- ADDED for low order models -has_cycling -- ADDED for low order models - -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -+ Feb 15th 2017 :: filter updates. Revision: 11160 -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -The postassim diagnostics file was being incorrectly written after -posterior inflation was applied. It is now written immediately after -the assimilation update, and then posterior inflation, if enabled, -is applied. - -Sampling Error Correction now reads data from a single netcdf file -for any ensemble size. To add other sizes, a program can generate -any ensemble size and append it to this file. The default file is -currently in system_simulation: - -system_simulation/work/sampling_error_correction_table.nc - -Filter and PMO no longer need the "has_cycling" flag. - ------------------------------------------------------------------------------- -Changes to the filter_nml are : ------------------------------------------------------------------------------- - -has_cycling -- REMOVED for low order models - ------------------------------------------------------------------------------- -Changes to the perfect_model_obs_nml are : ------------------------------------------------------------------------------- - -has_cycling -- REMOVED for low order models - -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -+ April 27th 2017 :: diagnostic file changes. Revision: 11545 -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -Two additional Diagnostic Files (forecast and analysis) in Filter -which can be set with the namelist option (stages_to_write) - - input - writes out mean and sd if requested. for low order models - mean and sd are only inserted into restart files with a - single time step. - - forecast - contains the forecast and potentially the mean and sd for the, - this is mostly important for lower order models which cycle - - preassim - before assimilation - * No Inflation: same as forecast - * Prior Inf: the inflated ensemble and damped prior inf - * Post Inf: same as forecast - * Prior and Post Inf: the inflated ensemble and damped prior inf - - postassim - after assimilation (before posterior infation) - * No Inflation: same as analysis - * Prior Inf: same as analysis - * Post Inf: assimilated ensemble and damped posterior inflation - * Prior and Post Inf: assimilated ensemble and damped posterior inflation - - analysis - after assimilation and before potentially update posterior inflation ensemble - and updated prior inf - * No Inflation: assimilated ensemble - * Prior Inf: assimilated ensemble and updated prior inf - * Post Inf: post inflated ensemble and updated posterior inflation - * Prior and Post Inf: post inflated ensemble and updated prior inf and posterior - inflation - - output - a single time step of the output ensemble and potentially updated prior inf - and posterior inflation - - -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -+ May 5th 2017 :: major changes to model_mod interfaces. Revision: 11615 -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -A long-awaited overhaul of the model_mod interfaces. All models which are -in our subversion repository and are supported in the Manhattan release -have been updated to match the new interfaces. If you have model_mods with -extensive changes, our recommendation is to diff your changes with the version -you checked out and insert those changes into the new version. The changes for -this update are unfortunately extensive. - -The detailed list of changes: - -model_mod::get_state_meta_data() is no longer passed an ensemble_handle as the -first argument. it should not do vertical coordinate conversion. that will be -done as a separate step by convert_vertical_state() - -model_mod::vert_convert is replaced by convert_vertical_state() and convert_vertical_obs() -Any vertical conversion code that was in get_state_meta_data should be moved -to convert_vertical_state() which has access to the state vector index, so the -code should move easily. - -model_mod::query_vert_localization_coord is no longer a required interface -model_mod::get_close_maxdist_init is not longer a required interface -model_mod::get_close_obs_init is not longer a required interface - -model_mod::get_close_obs has a different calling convention and is split into -get_close_obs() and get_close_state(). the close obs routine is passed both the -obs types and quantities, and the close state routine is passed both the -state quantities and the state index, for ease in vertical conversion if needed. - -model_mod::nc_write_model_vars() is deprecated for now; it may return in a -slightly different form in the future. - -model_mod::nc_write_model_atts() is now a subroutine with different arguments. -it should now only write any global attributes wanted, and possibly some grid -information. it should NOT write any of the state variables; those will be -written by DART routines. - -model_mod::get_model_size() needs to return an i8 for the size. - -A new module "default_model_mod" supplies default routines for any required -interfaces that don't need to be specialized for this model. - -A new module "netcdf_utilities_mod" can do some simple netcdf functions for -you and we plan to add many more over the next couple months. - -model_mod::get_model_time_step has been replaced by shortest_time_between_assimilations() -since in fact it has always controlled the minimum time filter would request a model advance -and never had anything to do with the internal time step of the dynamics of the model. - -We have removed 'output_state_vector' from the namelist of all model_mods since -we no longer output a single 1d vector. all i/o is now in netcdf format. - -Models now have more control over when vertical conversion happens - on demand -as needed, or all up front before assimilation. - -Models that were doing vertical conversion in get_state_meta_data should set: -&assim_tools_nml - convert_all_state_verticals_first = .true. - convert_all_obs_verticals_first = .true. - -Models which were not should set: - convert_all_state_verticals_first = .false. - convert_all_obs_verticals_first = .true. - -The location_mod::vert_is_xxx() routines have become a single is_vertical(loc, "string") where -string is one of: "PRESSURE", "HEIGHT", "SURFACE", "LEVEL", "UNDEFINED", "SCALE_HEIGHT" - -Models doing vertical localization should add a call to set_vertical_localization_coord() -in their static_init_model() routine to tell dart what vertical coordinate system they -are expecting to convert to for vert localization - -Most path_names_xxx files have been updated to add additional modules. compare against -what is checked out to see the differences. - -Some of the internal changes include pulling common code from the locations -modules into a location_io_mod which contains common functions for creating -and writing 'location' variables for any location type. - -QTY_RAW_STATE_VARIABLE is redundant and was shortened to QTY_STATE_VARIABLE - -Many utility programs use the template/model_mod.f90 because they don't -depend on any model-specific functions. this file was also updated to -match the new interfaces. - -The obs_impact facility is enabled in the assim_tools namelist. you can -use the obs_impact_tool to construct a table which prevents one class of -observations from impacting another class of state. - -Sampling Error Correction now reads the values it needs from a single -netcdf file found in assimilation_code/programs/gen_sampling_err_table/work. -copy it to the same directory as where filter is running. all ensemble -sizes which were previously in final_full.XX files are included, and there -is a tool to generate and append to the file any other ensemble size required. - -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -+ Jun 2nd 2017 :: tutorial, DART_LAB, and various updates. Revision: 11696 -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -- bring the DART tutorial pdf slides up to date with the current release. - -- include new GUIs with adaptive inflation options in DART_LAB: - oned_model_inf.m - run_lorenz_96_inf.m - -- added the lorenz_96_2scale model - additional kinds of - QTY_SMALL_SCALE_STATE and QTY_LARGE_SCALE_STATE added as required. - -- add useful attributes to the variables in the diagnostic files - -- updates and minor bug fixes to the matlab diagnostic scripts - -- updates to the default input.nmls for models - -- updates to the cam-fv shell scripts to work with the cesm 2.0 framework - -- updates to the cam-fv model_mod for support of cam-chem variables - Added more QUANTITIES/KINDS for chemistry species. - Removed support for 'stand-alone' CAM and CAM-SE (cam-se will be a separate 'model'). - -- major bug fix in the simple_advection model_mod: Fixed an error with - the layout of the state vector. - -- obs_def_radar_mod: Fixed a serious bug in the fall velocity forward operator. - If the fall speed field is not in the state the test for a bad istatus from - the interpolate() call was looking at the wrong variable and returning ok - even if interpolate() had set bad values. - -- bug fix in the wrf model_mod for fields which have a vertical stagger - -- fix to the makefiles for the GSI2DART observation converter - -- added additional netcdf and location utility routines - -- various fixes to documentation and test code - -- renamed QTY_RAW_STATE_VARIABLE to QTY_STATE_VARIABLE (RAW is redundant) - -- direct_netcdf_mod: Renamed 'limit_mem' to 'buffer_state_io'. - buffer_state_io is now a logical that states if a variable that tells - DART it it should read and write variables all at once or variable-by-variable. - -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -+ June 27rd 2017 :: CICE 5, model_mod_check, tutorial. Revision: 11770 -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -Updated support for CICE5. - -Updated support for model_mod_check - now compatible with netCDF input files, -input is through [input,output]_state_files namelist variable -(variables renamed). - -Ensured consistency between low-order namelists and the updated DART tutorial. -Updated documentation of many namelists. More to come. - -location_mod: namelist variable 'maintain_original_vert' was deprecated, -it is now removed. You must remove it from your existing namelists or -DART will error out right away. - -obs_diag: namelist variables 'rat_cri' and 'input_qc_threshold' have been -deprecated for years, they have been removed. You must remove them from -your existing namelists or obs_diag will error out right away. - -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -+ July 7th 2017 :: cam-fv, MPAS-ATM scripts, single file i/o. Revision: 11807 -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -models/mpas_atm: scripts completely revised for the Manhattan release. -Many thanks to Soyoung Ha and Ryan Torn for the contributed code. - -cam-fv: scripts and model_mod.f90 updated for cesm2_0_beta05. - -Single File I/O: - - Now we are able to run 'single_file_in' and 'single_file_out' with MPI. - - single_file_io_mod.f90 has been removed and its functionality has been moved - to direct_netcdf_mod.f90. - - single_file_io_mod.f90 has been removed from all of the path_names_* files - in the repository. (Remove it from any private path_names_* files.) - -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -+ July 18 2017 :: bug fixes, documentation updates. Revision: 11830 -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -- fixed bug in obs_impact_tool when generating the run-time table. specifying - a generic quantity resulted in selecting the wrong specific obs types. - -- fixed a bug that would not allow filter to start from a single ensemble member - if single_file_in was true. - -- updates to HTML documentation especially for types/quantities (replacing kinds) - -- updates to input.nml namelists, code comments, and shell scripts where - names changed from 'restart' to 'state' for input and output files. - -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -+ Aug 2 2017 :: single filenames, random distributions, bug fixes. Revision: 11864 -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -- added code to support listing input and output filenames directly in the - namelist instead of having to go through an indirect text file. most useful - for programs that take a single input and output file, but works for all cases. - -- bug fix in location_io_mod.f90 that affected obs_seq_to_netcdf (error in adding - vertical location types to output file). - -- fix to convert_gpsro_bufr.f90 converter (GPS obs from BUFR files) that failed - if r8 defined to be r4. - -- added draws from gamma, inverse gamma, and exponential distributions to the - random sequence module. - -- various updates to the CAM scripts to work more smoothly with the most - recent CIME changes and DART Manhattan updates. - -- added QTY_CWP_PATH and QTY_CWP_PATH_ZERO to the default quantities list for - the obs_def_cwp_mod.f90 forward operator. - -- improved some error messages in the diagnostic matlab scripts - -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -+ Oct 17 2017 :: MPAS ATM bug fix, various other updates. Revision: 12002 -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -- Fixed a bug in the MPAS-ATM model_mod that affected surface observations, - in particular altimeter obs. also fixed a bug in the vertical conversion - if using 'scale height' as the vertical localization type. - -- Fixed a bug in the CAM-FV model_mod which might have excluded observations - with a vertical coordinate of height (meters) which were in fact below the - equivalent highest_obs_pressure_Pa namelist setting. also fixed a possible - memory leak. - -- Added two new modules: options_mod.f90 and obs_def_utilities_mod.f90 - this was required so we didn't have circular dependencies in our modules - as we reused common code in more places. - - We have updated all the path_names* files which are in the repository. - if you have your own path_names files you may need to add these new modules - to your path lists. - - assimilation_code/modules/utilities/options_mod.f90 - observations/forward_operators/obs_def_utilities_mod.f90 - -- Removed QTY_SURFACE_TEMPERATURE from the default obs quantities list - and added QTY_2M_SPECIFIC_HUMIDITY. QTY_2M_TEMPERATURE exists for - atmospheric models, and QTY_SKIN_TEMPERATURE and QTY_SOIL_TEMPERATURE - exist for other models. if you were using QTY_SURFACE_TEMPERATURE - please replace it with the corresponding other temperature quantity. - -- Updated and improved the observation converter for ionospheric observations - from the COSMIC GPS satellite. - -- Updated the CAM-FV scripts for cesm2_0_beta05. - -- Updated the Matlab diagnostics documentation. 'help DART' or 'doc DART' - will give an overview of the available Matlab diagnostics shipped with the - dart distribution. - -- Added the observation type COSMIC_ELECTRON_DENSITY to the obs_def_upper_atm_mod - -- dart_to_clm and clm_to_dart were resurrected to correctly handle conversions - for the SWE (snow water equivalent) field. - -- Updated the channel and column location modules to be compatible with - the current required interfaces. - -- Updated the model_mod_check.f90 program (most often used when porting - DART to a new model). there is now more control over exactly which - tests are being run. updated the nml and html documentation files to - match the current code and describe the tests in more detail. - -- Fixed a misleading status message in the obs_sequence_tool when all obs - are excluded by the min/max lon/lat box namelist items. the incorrect - message blamed it on observation height instead of the bounding box. - -- Added some additional debugging options to the mpi utilities module. - if you have problems that appear to be MPI related, contact us for - more help in enabling them. - -- Improved some error messages in location_io_mod and state_structure_mod - -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -+ Nov 21 2017 :: 1D obs_diag fix, 1D power forward operator Revision: 12138 -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -- fixed a bad URL reference in tutorial section 18 - -- fixed a crash with the 1D version of the observation diagnostics program - when including identity observations. - -- all models with a workshop_setup.csh now build the same set of programs. - (some/most did not build obs_diag - which is used in the tutorial) - -- added a 1D obs-to-a-power forward operator. - -- updates to the matlab plotting routines for NetCDF observation formats - -- World Ocean Database (WOD) converter supports partial year conversions - and 2013 file formats. - -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -+ Nov 22 2017 :: minor updates for DA challenge files Revision: 12144 -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -- added obs_seq.in.power to the Lorenz 96 directory - -- added new obs types to the workshop version of the input.nml assimilation list - -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -+ Dec 01 2017 :: ROMS scripting, debugging aids Revision: 12166 -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -- Added an option to the ROMS model scripting to advance the model ensemble -members in parallel using a job array. - -- Updated the DART_LAB Matlab GUIs to log a history of the settings and -results. - -- Added a debug option to the filter namelist, 'write_obs_every_cycle', -to output the full obs_seq.final during each cycle of filter. -(Very slow - use only when debugging a filter crash.) - -- Allow the test grid in model_mod_check to cross the prime meridian for testing -longitude interpolation in grids that cross the 360/0 line. - -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -+ Mar 01 2018 :: ROMS, MMC, PMO, MPAS_ATM debug, etc Revision: 12419 -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -- Fix a debug message in the MPAS ATM model which might have caused a buffer -overflow crash when formatting a message for a larger ensemble size. - -- Update the ROMS shell scripts to support PBS, SLURM, as well as LSF. -Update the ROMS model_mod html documentation. - -- Update the default CAM-FV input.nml to have more realistic values -for the highest observation assimilated, and for where the ramp starts -that decreases the increments at the model top. If running with a higher -model top than the default check these items carefully. - -- Fixed variable type for 'time' variables we create in diagnostic files - -- Miscellaneous minor Bug fixes: - - Print format wider for fractional levels in threed_sphere locations - - Fixed a deallocate call at program shutdown time - - Fixed an indexing problem computing cam-fv U_WIND observations if the - observation used HEIGHT as the vertical coordinate (very unusual). - - Fixed grid creation bug in a test program used with model_mod_check. - Now uses correct spacing for grids in the x,y coordinates. - - Fixed an allocate problem in a test interpolate routine. - -- Add surface pressure to the default state list in the wrf work/input.nml - -- developer_tests/test_dart.csh can run PMO for more models. required -updates to the work/input.nml in several directories (wrf, cm1, POP, -mpas_atm) to match the current namelist. - -- several model_mod_check programs were combined into a single version -that allows for selection of individual tests. many of the input.nml -models/xxx/work/input.nml files have either had a &model_mod_check_nml -section added or updated to match the updated interface. - -- the DART QTYs are now available via the state structure in the wrf -and clm model_mods. - -- support the NAG compiler better. (contact dart@ucar.edu for more -help if you want to use this compiler. some hand work is still needed.) - -- streamlined the debug output from the state_structure_info() call to -avoid replicating information that was the same for all variables. - -- minor formatting change to the dart log file output for the list of -observation types being assimilated, evaluated, and using precomputed -forward operators. - -- fixed an uninitialized variable in the BGRID model code in a routine -that isn't normally used. - -- Updated the threed_sphere location module documentation with some usage -notes about issues commonly encountered. - -- Fixed an incorrect test when printing out a log message describing if the -inflation would be variance-adaptive or not. - -- Change the location of the POP MDT reference file to be relative to the -current run directory and not an absolute file location on cheyenne. - -- Make the ROMS, CM1, and POP model_mod log namelist information to the -namelist log file and not the main DART log file. - -- Updated several html documentation files, including the template/model_mod.html -which describes the current model_mod required interfaces. - -- Updated the instructions for the GSI to DART obs converter to suggest some -needed compiler flags in certain cases. - -- Updated the location module test programs. - -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -+ May 21 2018 :: enhanced inflation option, scripting Revision: 12591 -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -- Enhanced inflation algorithm added. See the filter_mod.html for new -documentation on this option. - -- Updated WRF scripts for the Manhattan release. - -- Obs_diag reports statistics on DART QC 8, observation failed vertical -conversion. Matlab scripts also updated to support QC 8. - -- New parallel conversion scripts for GPS Radio Occultation observations and - NCEP prepbufr conversions. - -- Further updates to documentation files to change KIND to QTY or Quantity. - -- Documented required changes when moving from the Lanai/Classic release to -Manhattan in documentation/html/Manhattan_diffs_from_Lanai.html - -- Expanded the routines in the netcdf_utilities_mod.f90 - -- Add an ensemble handle parameter to the 6 ensemble manager routines -where it was missing. - -- The advance_time program can read/generate CESM format time strings -(YYYY-MM-DD-SSSSS). - -- Fixed a bug in the netcdf read routines that under certain circumstances -could report an array was using the unlimited dimension incorrectly. - -- Removed the option to try to bitwise reproduce Lanai results; due to the -number of changes this is no longer possible. - -- Minor bug fixes to the (seldom used) perturb routines in the WRF -and MPAS_ATM model_mod.f90 files. (used to add gaussian noise to a -single model state to generate an ensemble; this is never the recommended -method of starting a new experiment but the code remains for testing -purposes.) - -- Several remaining model-specific model_mod_check programs were -removed in favor of a single common program source file. - -- Keep filter_mod.dopplerfold.f90 in sync with filter_mod.f90, -and assim_tools_mod.pf.f90 in sync with assim_tools_mod.f90. - -- Removed makefiles for the obsolete trans_time program. - -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -+ Jun 18 2018 :: CAM/CESM 2.0, DART QC 8, closest_member_tool Revision: 12682 -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -- Support for CAM-FV assimilations in the CESM 2.0 release. See - documentation in models/cam-fv/doc/README_cam-fv for details. - -- obs_diag and matlab scripts updated to report statistics on DART QC 8, - observation failed vertical conversion - -- Updates to fix minor problems with the new WRF scripts - -- Added the 'inf_sd_max_change' namelist item to all input.nml files for - the enhanced inflation option - -- Revival of the closest_member_tool, which now runs in parallel on - all ensemble members at one time. This tool can be used as a template - for any other tools which need to process something for all ensemble - members in parallel. - -- Revival of the fill_inflation_restart tool as a Fortran 90 program. - Using ncap2 is still possible, but if the correct version is not - installed or available this tool can be used. - -- Added more functions to the netcdf_utilities_mod.f90 - -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -+ Aug 03 2018 :: performance fix for distributed mean Revision: 12758 -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -- Important performance fix if model does vertical conversion for localization. - Results were not wrong but performance was poor if 'distribute_mean = .true.' - was selected in the &assim_tools_nml namelist. - - Now distributing the mean runs in close to the non-distributed time and uses - much less memory for large models. This only impacts models which do a vertical - conversion of either the observation or state vertical coordinate for localization - AND which set &assim_tools_nml :: distribute_mean = .true. to use less memory. - - When using a distributed mean "convert_all_obs_verticals_first = .true." should - be set. If your observations will impact most of the model state, then - "convert_all_state_verticals_first = .true.' can also be set. - -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -+ Apr 30 2019 :: cam-fv refactor, posteriors optional, QC 8 Revision: 13138 -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -- The CAM Finite Volume (cam-fv) model_mod.f90 has undergone substantial - refactoring to improve simplicity and remove code for unsupported CAM variants - while also supporting WACCM and WACCM-X. Namelist changes will be required. - -- CAM-FV setup and scripting support added for CESM 2.1, including advanced - archiving and compression - -- fix for WRF's wind direction vectors when using the Polar Stereographic - map projection. Thanks to Kevin Manning for the fix. - -- Add filter namelist option to avoid calling the posterior forward operators - and to not create those copies in the obs_seq.final file. - -- Use less memory if writing ensemble member values into the obs_seq.final file. - -- added a DART QC of 8 for failed vertical conversions - -- updated Matlab scripts support QC=8 and no posterior in obs sequence files. - -- sampling error correction table now has all ensemble sizes between 3 and 200 - -- closest_member_tool can be compiled with other MPI targets - -- COSMIC_ELECTRON_DENSITY has been moved from obs_def_gps_mod.f90 to - obs_def_upper_atm_mod.f90, which has new quantities for - ION_O_MIXING_RATIO and ATOMIC_H_MIXING_RATIO - -- obs_converters/gps/convert_cosmic_ionosphere.f90 has a test dataset - -- support for NAG compiler - -- fixed Intel compiler bug in lorenz_96 comparing long integers to integer loop indices - -- get_maxdist() now a required routine all location modules - -- Default routines now create a time variable as time(time) to allow multiple - files to be concatenated along the unlimited dimension more easily. Also - conforms to the netCDF convention for coordinate dimensions. - -- obs_impact_tool handles a continuum of values, not just discrete 0 or 1. - -- fill_inflation_restart now produces files with names consistent with filter defaults. - -- expanded functionality in xyz_location_mod.f90 - -- Removed 'slow' sorting routines from sort_mod.f90 - -- replacing some repeated native netCDF library calls with routines from - the netcdf_utilities_mod.f90 - -- Updated dewpoint equation to avoid dividing by zero given a very unlikely - scenario (r12832) - -- More efficient implementation of adaptive inflation - -- Yongfei Zhang and Cecilia Bitz added improvements to the CICE model and - observation converters and forward operators. These changes also use the - locations of the 'new' glade filesystem. They used CESM tag: cesm2_0_alpha06n - -- Worked with Yongfei Zhang to remove prototype codes and more completely - document observation converters and data sources for cice assimilation. - -- removed 'allow_missing_in_clm' flag from the &assim_tools_nml namelist in - the CICE work directory. The flag moved to a different namelist and the - CICE model doesn't care about it. - -- increased the maximum number of input files to obs_diag from 100 to 10000. - -- Updated the developer_tests to include more cases. - -- Updated oned/obs_diag.f90 to support 'obs_seq.out' files. - -- Better error and informational messages in various routines. - -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -+ Nov 20 2019 :: FESOM,NOAH-MP model support, better testing Tag: v9.8.0 -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -- first release entirely from GIT - -- fixed bug in fill_inflation_restart tool which used the prior inflation mean - and sd for both prior and posterior inflation files. now correctly uses the - posterior mean/sd if requested. - -- fixed a typo in the location test script that prevented it from running - -- additional functionality in the quad interpolation code, now supports grids - which start at 90 (north) and end at -90 (south). - -- if possible, send shorter MPI messages. improves performance on some platforms - and MPI implementations. - -- add explicit call to initalize_utilities() where it was missing in a couple of - the WRF utility routines. - -- added an example of how to use a namelist to the text_to_obs.f90 observation - converter program. - -- Removing the clamping messages in 'clamp_variable()' of clamped values - -- changed argument names using reserved keywords. - state_vector_io_mod:read_state() 'time' to 'model_time' - random_seq_mod:random_gamma() 'shape' to 'rshape', 'scale' to 'rscale'. - random_seq_mod:random_inverse_gamma() 'shape' to 'rshape', 'scale' to 'rscale'. - obs_def_mod:init_obs_def() 'kind' to 'obkind', 'time' to 'obtime' - obs_utilities_mod: 'start' to 'varstart', 'count' to 'varcount' - -- The FESOM model is now Manhattan-ready. - -- The noah model is now Manhattan-ready and may be used with NOAH-MP. - -- bugfixed references to the 'documentation' directory that was renamed - 'docs' to comply with GitHub Pages. - -- improved test_dart.csh functionality. - -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -+ MMM DD YYYY :: summary of changes in next Manhattan update Tag: vX.Y.Z -++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000..e7fabf2316 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,885 @@ +DART software - Copyright UCAR. This open source software is provided +by UCAR, "as is", without charge, subject to all terms of use at +http://www.image.ucar.edu/DAReS/DART/DART_download + +---- + +This file documents the most user-visible changes to the DART code. +It is not intended to document every change, but instead is intended +to inform people what features are now available or have been removed. +Detailed changes are always available through the version control framework. + +## SEE THE BOTTOM OF THIS FILE FOR THE MOST RECENT CHANGES! + +---- +---- + +## Jan 13th 2017 :: rma_fixed_filenames merge changes. Revision: 10902 + +Specific namelist changes include: + +1. Earlier versions of the RMA branch code supported both direct NetCDF + reads/writes and the original binary/ascii DART format restart files. + As of the next update DART format files are no longer supported. All + I/O is NetCDF only. If your model does not use NetCDF you will still + need a model_to_dart and dart_to_model converter; otherwise all DART + programs read the model's NetCDF files directly. The namelist options + related to selecting direct netcdf I/O have been removed. + +1. Diagnostic and state space data (such as inflation, mean and sd + information) that were previously stored in {Prior,Posterior}_Diag.nc + are now broken up into multiple files and have fixed filenames. This + decreases the IO time for diagnostic output and reduces the number of + namelist options. + +1. There is no longer support for observation space inflation + (i.e. inf_flavor = 1). Contact us at dart@ucar.edu if you have an + interest in using this option. + +#### Changes to the filter_nml are : + +- `restart_in_file_name` has been replaced with `input_restart_file_list`. + The namelist must contain one or more file names, + each of which is a textfile containing a list of N + NetCDF restart files, one per line for each ensemble member. + For models with multiple domains (e.g. nested WRF or CLM) + you must specify a listfile for each domain. + +- `restart_out_file_name` has been replaced with `output_restart_file_list`. + Same format as `input_restart_file_list`. + +- `inf_in_file_name` REMOVED, now have fixed names of the form + input_{prior,posterior}inf_{mean,sd}.nc + +- `inf_out_file_name` REMOVED, now have fixed names of the form + output_{prior,posterior}inf_{mean,sd}.nc. + +- `inf_diag_filename` REMOVED + +- `inf_output_restart` REMOVED, inflation restarts will be written + out if inflation is turned on + +- `output_inflation` REMOVED, inflation diagnostic files will be written + if inflation is turned on + +- `stages_to_write` There is more control over what state data + to write. Options are at stages : 'input', 'preassim', postassim', 'output'. + Stages preassim and postassim will output state data originally contained + within the copies of `Prior_Diag.nc` and `Posterior_Diag.nc`. + See rma_doc/rma.html for details on the filename conventions. + For example, running filter with prior inflation enabled with stage + 'preassim' enabled will produce files with names: + - preassim_member_####.nc + - preassim_{mean,sd}.nc + - preassim_priorinf_{mean,sd}.nc + +- `write_all_stages_at_end` important for large models - all output file + I/O is deferred until the end of filter, but will use more memory to store + the data. More detailed info is in rma_doc/rma.html + +- `output_restart_mean` renamed output_mean + +- `output_restart` renamed output_restarts + +- `direct_netcdf_{read,write}` REMOVED, always true + +- `restart_list_file` renamed input_restart_file_list + +- `single_restart_file_in` renamed single_file_in + +- `single_restart_file_out` renamed single_file_out + +- `add_domain_extension` REMOVED + +- `use_restart_list` REMOVED + +- `overwrite_state_input` REMOVED, equivalent functionality + can be set with `single_restart_file_in = single_restart_file_out` + +#### Changes to the perfect_model_obs_nml are : + +- `restart_in_filename` renamed `restart_in_file_names` takes a NetCDF + file. For multiple domains you can specify a list. + +- `direct_netcdf_{read,write}` REMOVED, always true + +#### Changes to the state_space_diag_nml are : + +- `single_file` REMOVED, diagnostic files are now controlled + in `filter_nml` with `stages_to_write` + +- `make_diagnostic_files` REMOVED, no longer produce + original `Prior_Diag.nc` and `Posterior_Diag.nc` + +- `netCDF_large_file_support` REMOVED, always true + +#### Changes to the state_vector_io_nml are : + +- `write_binary_restart_files` REMOVED + +#### Changes to the ensemble_manager_nml are : + +- `flag_unneeded_transposes` -- REMOVED + +#### Changes to the integrate_model_nml are : + +- `advance_restart_format` -- REMOVED, only supporting NetCDF format. + +#### Scripting with CESM + +See `models/cam-fv/scripts_cesm1_5/assimilate.csh` for an example of how to +handle the new filename conventions. + +``` +(To help find things: input_priorinf_mean output_priorinf_mean ) +{in,out}put_{prior,post}inf_{mean,sd}.nc ARE in use; + Search for stage_metadata%filenames turned up + interface set_file_metadata + module procedure set_explicit_file_metadata + module procedure set_stage_file_metadata + + ! stage_name is {input,preassim,postassim,output} + ! base_name is {mean,sd,{prior,post}inf_{mean,sd}} from filter/filter_mod.f90. + write(string1,'(A,''.nc'')') trim(stage_name)//'_'//trim(base_name) + file_info%stage_metadata%filenames(my_copy,1) = trim(string1) + + This shows where inflation file names are defined. + > grep -I set_file_metadata */*.f90 | grep inf + filter/filter_mod.f90: + call set_file_metadata(file_info, PRIOR_INF_MEAN, stage, 'priorinf_mean', 'prior inflation mean') + call set_file_metadata(file_info, PRIOR_INF_SD, stage, 'priorinf_sd', 'prior inflation sd') + call set_file_metadata(file_info, POST_INF_MEAN, stage, 'postinf_mean', 'posterior inflation mean') + call set_file_metadata(file_info, POST_INF_SD, stage, 'postinf_sd', 'posterior inflation sd') + + subroutine set_member_file_metadata(file_info, ens_size, my_copy_start) + call set_file_metadata(file_info, icopy, stage_name, base_name, desc, offset) + + subroutine set_stage_file_metadata(file_info, copy_number, stage, base_name, desc, offset) + write(string1,'(A,''.nc'')') trim(stage_name)//'_'//trim(base_name) + + subroutine set_explicit_file_metadata(file_info, cnum, fnames, desc) + file_info%stage_metadata%filenames(cnum,idom) = trim(fnames(idom)) + file_info%stage_metadata%file_description(cnum,idom) = trim(string1) + + function construct_file_names(file_info, ens_size, copy, domain) + write(construct_file_names, '(A, ''_member_'', I4.4, A, ''.nc'')') & + trim(file_info%root_name), copy, trim(dom_str) + +Also see + harnesses/filename_harness/files: ENS_MEAN_COPY PriorDiag_mean.nc +``` + +#### ADDITIONAL NOTES : + +1. currently the closest_member_tool is broken but plans on being fixed soon. +1. restart_file_tool and most model_to_dart/dart_to_model programs have been + deprecated, since DART formated restarts are no longer supported. +1. some programs such as model_mod_check have not been fully tested and need + to be exercised with the new naming conventions. + + +------------------------------------------------------------------------------ +## Feb 15th 2017 :: rma_single_file merge changes. Revision: 11136 + +Filter and PMO can now run with multiple cycles for low order models. The output +for this is only supported with single file output (members, inflation, mean, sd +are all in the same file). + +Added matlab support for diagnostics format in lower order models. + +#### Changes to the filter_nml are : + +- `output_restart` RENAMED to `output_members` +- `restart_in_file_name` RENAMED to `input_state_file_list` +- `restart_out_file_name` RENAMED to `output_state_file_list` +- `single_restart_file_in` RENAMED to `single_file_in` +- `single_restart_file_out` RENAMED to `single_file_out` + +- `input_state_files` ADDED - not currently working +- `output_state_files` ADDED - not currently working + +- `has_cycling` ADDED for low order models + +#### Changes to the perfect_model_obs_nml are : + +- `start_from_restart` RENAMED `read_input_state_from_file` +- `output_restart` RENAMED `write_output_state_to_file` +- `restart_in_file_name` RENAMED `input_state_files` +- `restart_out_file_name` RENAMED `output_state_files` +- `single_file_in` ADDED for low order models +- `single_file_out` ADDED for low order models +- `has_cycling` ADDED for low order models + + +------------------------------------------------------------------------------ +## Feb 15th 2017 :: filter updates. Revision: 11160 + +The postassim diagnostics file was being incorrectly written after +posterior inflation was applied. It is now written immediately after +the assimilation update, and then posterior inflation, if enabled, +is applied. + +Sampling Error Correction now reads data from a single netcdf file +for any ensemble size. To add other sizes, a program can generate +any ensemble size and append it to this file. The default file is +currently in `system_simulation`: + +`system_simulation/work/sampling_error_correction_table.nc` + +Filter and PMO no longer need the "has_cycling" flag. + +#### Changes to the filter_nml are : + +- `has_cycling` REMOVED for low order models + +#### Changes to the perfect_model_obs_nml are : + +- `has_cycling` REMOVED for low order models + + +------------------------------------------------------------------------------ +## April 27th 2017 :: diagnostic file changes. Revision: 11545 + +Two additional Diagnostic Files (forecast and analysis) in Filter +which can be set with the namelist option (stages_to_write) + +- **input** writes out mean and sd if requested. + - For low order models, mean and sd are only inserted into restart + files with a single time step. + +- **forecast** + - contains the forecast and potentially the mean and sd for the, + this is mostly important for lower order models which cycle + +- **preassim** before assimilation + - No Inflation: same as forecast + - Prior Inf: the inflated ensemble and damped prior inf + - Post Inf: same as forecast + - Prior and Post Inf: the inflated ensemble and damped prior inf + +- **postassim** after assimilation (before posterior infation) + - No Inflation: same as analysis + - Prior Inf: same as analysis + - Post Inf: assimilated ensemble and damped posterior inflation + - Prior and Post Inf: assimilated ensemble and damped posterior inflation + +- **analysis** after assimilation and before potentially update posterior inflation ensemble and updated prior inf + - No Inflation: assimilated ensemble + - Prior Inf: assimilated ensemble and updated prior inf + - Post Inf: post inflated ensemble and updated posterior inflation + - Prior and Post Inf: post inflated ensemble and updated prior inf and posterior inflation + +- **output** + - a single time step of the output ensemble and potentially updated prior inf and posterior inflation + + +------------------------------------------------------------------------------ +## May 5th 2017 :: major changes to model_mod interfaces. Revision: 11615 + +A long-awaited overhaul of the model_mod interfaces. All models which are +in our subversion repository and are supported in the Manhattan release +have been updated to match the new interfaces. If you have model_mods with +extensive changes, our recommendation is to diff your changes with the version +you checked out and insert those changes into the new version. The changes for +this update are unfortunately extensive. + +The detailed list of changes: + +`model_mod::get_state_meta_data()` is no longer passed an ensemble_handle as the +first argument. it should not do vertical coordinate conversion. that will be +done as a separate step by `convert_vertical_state()` + +`model_mod::vert_convert` is replaced by `convert_vertical_state()` and `convert_vertical_obs()` +Any vertical conversion code that was in `get_state_meta_data` should be moved +to `convert_vertical_state()` which has access to the state vector index, so the +code should move easily. + +`model_mod::query_vert_localization_coord` is no longer a required interface +`model_mod::get_close_maxdist_init` is not longer a required interface +`model_mod::get_close_obs_init` is not longer a required interface + +`model_mod::get_close_obs` has a different calling convention and is split into +`get_close_obs()` and `get_close_state()`. the close obs routine is passed both the +obs types and quantities, and the close state routine is passed both the +state quantities and the state index, for ease in vertical conversion if needed. + +`model_mod::nc_write_model_vars()` is deprecated for now; it may return in a +slightly different form in the future. + +`model_mod::nc_write_model_atts()` is now a subroutine with different arguments. +it should now only write any global attributes wanted, and possibly some grid +information. it should NOT write any of the state variables; those will be +written by DART routines. + +`model_mod::get_model_size()` needs to return an `i8` (a long integer) for the size. + +A new module `default_model_mod` supplies default routines for any required +interfaces that don't need to be specialized for this model. + +A new module `netcdf_utilities_mod` can do some simple netcdf functions for +you and we plan to add many more over the next couple months. + +`model_mod::get_model_time_step` has been replaced by `shortest_time_between_assimilations()` +since in fact it has always controlled the minimum time filter would request a model advance +and never had anything to do with the internal time step of the dynamics of the model. + +We have removed `output_state_vector` from the namelist of all model_mods since +we no longer output a single 1d vector. all i/o is now in netcdf format. + +Models now have more control over when vertical conversion happens - on demand +as needed, or all up front before assimilation. + +Models that were doing vertical conversion in `get_state_meta_data` should set: +``` +&assim_tools_nml + convert_all_state_verticals_first = .true. + convert_all_obs_verticals_first = .true. + +Models which were not should set: + convert_all_state_verticals_first = .false. + convert_all_obs_verticals_first = .true. +``` + +The `location_mod::vert_is_xxx()` routines have become a single `is_vertical(loc, "string")` where +string is one of: "PRESSURE", "HEIGHT", "SURFACE", "LEVEL", "UNDEFINED", "SCALE_HEIGHT" + +Models doing vertical localization should add a call to `set_vertical_localization_coord()` +in their `static_init_model()` routine to tell dart what vertical coordinate system they +are expecting to convert to for vert localization + +Most `path_names_xxx` files have been updated to add additional modules. compare against +what is checked out to see the differences. + +Some of the internal changes include pulling common code from the locations +modules into a `location_io_mod` which contains common functions for creating +and writing 'location' variables for any location type. + +`QTY_RAW_STATE_VARIABLE` is redundant and was shortened to `QTY_STATE_VARIABLE` + +Many utility programs use the `template/model_mod.f90` because they do not +depend on any model-specific functions. this file was also updated to +match the new interfaces. + +The `obs_impact` facility is enabled in the `assim_tools` namelist. you can +use the `obs_impact_tool` to construct a table which prevents one class of +observations from impacting another class of state. + +Sampling Error Correction now reads the values it needs from a single +netcdf file found in `assimilation_code/programs/gen_sampling_err_table/work`. +Copy it to the same directory as where filter is running. All ensemble +sizes which were previously in `final_full.XX` files are included, and there +is a tool to generate and append to the file any other ensemble size required. + + +------------------------------------------------------------------------------ +## Jun 2nd 2017 :: tutorial, DART_LAB, and various updates. Revision: 11696 + +- bring the DART tutorial pdf slides up to date with the current release. + +- include new GUIs with adaptive inflation options in DART_LAB: + - `oned_model_inf.m` + - `run_lorenz_96_inf.m` + +- added the **lorenz_96_2scale** model - additional kinds of + `QTY_SMALL_SCALE_STATE` and `QTY_LARGE_SCALE_STATE` added as required. + +- add useful attributes to the variables in the diagnostic files + +- updates and minor bug fixes to the matlab diagnostic scripts + +- updates to the default input.nmls for models + +- updates to the **cam-fv** shell scripts to work with the CESM2.0 framework + +- updates to the **cam-fv** `model_mod` for support of `cam-chem` variables + Added more QUANTITIES/KINDS for chemistry species. + Removed support for 'stand-alone' **cam** and **cam-se** (**cam-se** will be a separate 'model'). + +- major bug fix in the **simple_advection** `model_mod`: Fixed an error with + the layout of the state vector. + +- `obs_def_radar_mod`: Fixed a serious bug in the fall velocity forward operator. + If the fall speed field is not in the state the test for a bad istatus from + the interpolate() call was looking at the wrong variable and returning ok + even if interpolate() had set bad values. + +- bug fix in the **wrf** model_mod for fields which have a vertical stagger + +- fix to the makefiles for the GSI2DART observation converter + +- added additional netcdf and location utility routines + +- various fixes to documentation and test code + +- renamed `QTY_RAW_STATE_VARIABLE` to `QTY_STATE_VARIABLE` (RAW is redundant) + +- `direct_netcdf_mod`: Renamed `limit_mem` to `buffer_state_io`. + `buffer_state_io` is now a logical that states if a variable that tells + DART it it should read and write variables all at once or variable-by-variable. + + +------------------------------------------------------------------------------ +## June 27rd 2017 :: CICE 5, model_mod_check, tutorial. Revision: 11770 + +- Updated support for CICE5. + +- Updated support for `model_mod_check` - now compatible with netCDF input files, + input is through [input,output]_state_files namelist variable (variables renamed). + +- Ensured consistency between low-order namelists and the updated DART tutorial. + Updated documentation of many namelists. More to come. + +- `location_mod`: namelist variable `maintain_original_vert` was deprecated, + it is now removed. You must remove it from your existing namelists or + DART will error out immediately. + +- `obs_diag`: namelist variables `rat_cri` and `input_qc_threshold` have been + deprecated for years, they have been removed. You must remove them from + your existing namelists or obs_diag will error out immediately. + + +------------------------------------------------------------------------------ +## July 7th 2017 :: cam-fv, mpas_atm scripts, single file i/o. Revision: 11807 + +- **mpas_atm**: scripts completely revised for the Manhattan release. + Many thanks to *Soyoung Ha* and *Ryan Torn* for the contributed code. + +- **cam-fv**: scripts and `model_mod.f90` updated for cesm2_0_beta05. + +Single File I/O: + +- Now we are able to run `single_file_in` and `single_file_out` with MPI. + +- `single_file_io_mod.f90` has been removed and its functionality has been moved + to `direct_netcdf_mod.f90`. + +- `single_file_io_mod.f90` has been removed from all of the `path_names_*` + files in the repository. (Remove it from any private `path_names_*` files.) + + +------------------------------------------------------------------------------ +## July 18 2017 :: bug fixes, documentation updates. Revision: 11830 + +- fixed bug in `obs_impact_tool` when generating the run-time table. specifying + a generic quantity resulted in selecting the wrong specific obs types. + +- fixed a bug that would not allow filter to start from a single ensemble member + if `single_file_in = .true.` + +- updates to HTML documentation especially for types/quantities (replacing kinds) + +- updates to `input.nml` namelists, code comments, and shell scripts where + names changed from `restart` to `state` for input and output files. + + +------------------------------------------------------------------------------ +## Aug 2 2017 :: single filenames, random distributions, bug fixes. Revision: 11864 + +- added code to support listing input and output filenames directly in the + namelist instead of having to go through an indirect text file. most useful + for programs that take a single input and output file, but works for all cases. + +- bug fix in `location_io_mod.f90` that affected `obs_seq_to_netcdf` (error in adding + vertical location types to output file). + +- fix to `convert_gpsro_bufr.f90` converter (GPS obs from BUFR files) that failed + if r8 defined to be r4. + +- added draws from gamma, inverse gamma, and exponential distributions to the + random sequence module. + +- various updates to the **cam** scripts to work more smoothly with the most + recent CIME changes and DART Manhattan updates. + +- added `QTY_CWP_PATH` and `QTY_CWP_PATH_ZERO` to the default quantities list for + the `obs_def_cwp_mod.f90` forward operator. + +- improved some error messages in the diagnostic matlab scripts + + +------------------------------------------------------------------------------ +## Oct 17 2017 :: mpas_atm bug fix, various other updates. Revision: 12002 + +- Fixed a bug in the **mpas_atm** `model_mod` that affected surface observations, + in particular altimeter obs. also fixed a bug in the vertical conversion + if using 'scale height' as the vertical localization type. + +- Fixed a bug in the **cam-fv** `model_mod` which might have excluded observations + with a vertical coordinate of height (meters) which were in fact below the + equivalent highest_obs_pressure_Pa namelist setting. also fixed a possible + memory leak. + +- Added two new modules: `options_mod.f90` and `obs_def_utilities_mod.f90` + this was required so we didn't have circular dependencies in our modules + as we reused common code in more places. + We have updated all the `path_names*` files which are in the repository. + if you have your own path_names files you may need to add these new modules + to your path lists. + - `assimilation_code/modules/utilities/options_mod.f90` + - `observations/forward_operators/obs_def_utilities_mod.f90` + +- Removed `QTY_SURFACE_TEMPERATURE` from the default obs quantities list + and added `QTY_2M_SPECIFIC_HUMIDITY`. `QTY_2M_TEMPERATURE` exists for + atmospheric models, and `QTY_SKIN_TEMPERATURE` and `QTY_SOIL_TEMPERATURE` + exist for other models. if you were using `QTY_SURFACE_TEMPERATURE` + please replace it with the corresponding other temperature quantity. + +- Updated and improved the observation converter for ionospheric observations + from the COSMIC GPS satellite. + +- Updated the **cam-fv** scripts for cesm2_0_beta05. + +- Updated the Matlab diagnostics documentation. 'help DART' or 'doc DART' + will give an overview of the available Matlab diagnostics shipped with the + dart distribution. + +- Added the observation type `COSMIC_ELECTRON_DENSITY` to the `obs_def_upper_atm_mod` + +- `dart_to_clm` and `clm_to_dart` were resurrected to correctly handle conversions + for the SWE (snow water equivalent) field. + +- Updated the channel and column location modules to be compatible with + the current required interfaces. + +- Updated the `model_mod_check.f90` program (most often used when porting + DART to a new model). there is now more control over exactly which + tests are being run. updated the nml and html documentation files to + match the current code and describe the tests in more detail. + +- Fixed a misleading status message in the `obs_sequence_tool` when all obs + are excluded by the min/max lon/lat box namelist items. the incorrect + message blamed it on observation height instead of the bounding box. + +- Added some additional debugging options to the mpi utilities module. + if you have problems that appear to be MPI related, contact us for + more help in enabling them. + +- Improved some error messages in `location_io_mod` and `state_structure_mod` + +------------------------------------------------------------------------------ +## Nov 21 2017 :: 1D obs_diag fix, 1D power forward operator Revision: 12138 + +- fixed a bad URL reference in tutorial section 18 + +- fixed a crash with the 1D version of the observation diagnostics program + when including identity observations. + +- all models with a `workshop_setup.csh` now build the same set of programs. + (some/most did not build obs_diag - which is used in the tutorial) + +- added a 1D obs-to-a-power forward operator. + +- updates to the matlab plotting routines for NetCDF observation formats + +- World Ocean Database (WOD) converter supports partial year conversions + and 2013 file formats. + +------------------------------------------------------------------------------ +## Nov 22 2017 :: minor updates for DA challenge files Revision: 12144 + +- added `obs_seq.in.power` to the Lorenz 96 directory + +- added new obs types to the workshop version of the `input.nml` assimilation list + +------------------------------------------------------------------------------ +## Dec 01 2017 :: ROMS scripting, debugging aids Revision: 12166 + +- Added an option to the ROMS model scripting to advance the model ensemble +members in parallel using a job array. + +- Updated the DART_LAB Matlab GUIs to log a history of the settings and results. + +- Added a debug option to the filter namelist, `write_obs_every_cycle`, +to output the full `obs_seq.final` during each cycle of filter. +(Very slow - use only when debugging a filter crash.) + +- Allow the test grid in `model_mod_check` to cross the prime meridian for testing +longitude interpolation in grids that cross the 360/0 line. + +------------------------------------------------------------------------------ +## Mar 01 2018 :: ROMS, MMC, PMO, mpas_atm debug, etc Revision: 12419 + +- Fix a debug message in the **mpas_atm** model which might have caused a buffer +overflow crash when formatting a message for a larger ensemble size. + +- Update the **ROMS** shell scripts to support PBS, SLURM, as well as LSF. +Update the ROMS model_mod html documentation. + +- Update the default **cam-fv** `input.nml` to have more realistic values +for the highest observation assimilated, and for where the ramp starts +that decreases the increments at the model top. If running with a higher +model top than the default check these items carefully. + +- Fixed variable type for `time` variables we create in diagnostic files + +- Miscellaneous minor Bug fixes: + - Print format wider for fractional levels in `threed_sphere` locations + - Fixed a deallocate call at program shutdown time + - Fixed an indexing problem computing **cam-fv** U_WIND observations if the + observation used HEIGHT as the vertical coordinate (very unusual). + - Fixed grid creation bug in a test program used with `model_mod_check`. + Now uses correct spacing for grids in the x,y coordinates. + - Fixed an allocate problem in a test interpolate routine. + +- Add surface pressure to the default state list in the **wrf** `work/input.nml` + +- `developer_tests/test_dart.csh` can run PMO for more models. required +updates to the `work/input.nml` in several directories +(wrf, cm1, POP, mpas_atm) to match the current namelist. + +- several `model_mod_check` programs were combined into a single version +that allows for selection of individual tests. many of the input.nml +`models/xxx/work/input.nml` files have either had a `&model_mod_check_nml` +section added or updated to match the updated interface. + +- the DART QTYs are now available via the state structure in the **wrf** +and **clm** `model_mod`s. + +- support the NAG compiler better. (contact dart@ucar.edu for more +help if you want to use this compiler. some hand work is still needed.) + +- streamlined the debug output from the `state_structure_info()` call to +avoid replicating information that was the same for all variables. + +- minor formatting change to the dart log file output for the list of +observation types being assimilated, evaluated, and using precomputed +forward operators. + +- fixed an uninitialized variable in the BGRID model code in a routine +that isn't normally used. + +- Updated the `threed_sphere` location module documentation with some usage +notes about issues commonly encountered. + +- Fixed an incorrect test when printing out a log message describing if the +inflation would be variance-adaptive or not. + +- Change the location of the POP MDT reference file to be relative to the +current run directory and not an absolute file location on cheyenne. + +- Make the ROMS, CM1, and POP model_mod log namelist information to the +namelist log file and not the main DART log file. + +- Updated several html documentation files, including the `template/model_mod.html` +which describes the current model_mod required interfaces. + +- Updated the instructions for the GSI to DART obs converter to suggest some +needed compiler flags in certain cases. + +- Updated the location module test programs. + +------------------------------------------------------------------------------ +## May 21 2018 :: enhanced inflation option, scripting Revision: 12591 + +- Enhanced inflation algorithm added. See the `filter_mod.html` for new +documentation on this option. + +- Updated WRF scripts for the Manhattan release. + +- `obs_diag` reports statistics on DART QC 8, observation failed vertical +conversion. Matlab scripts also updated to support QC 8. + +- New parallel conversion scripts for GPS Radio Occultation observations and + NCEP prepbufr conversions. + +- Further updates to documentation files to change KIND to QTY or Quantity. + +- Documented required changes when moving from the Lanai/Classic release to +Manhattan in `documentation/html/Manhattan_diffs_from_Lanai.html` + +- Expanded the routines in the `netcdf_utilities_mod.f90` + +- Add an ensemble handle parameter to the 6 ensemble manager routines +where it was missing. + +- The `advance_time` program can read/generate CESM format time strings +(YYYY-MM-DD-SSSSS). + +- Fixed a bug in the netcdf read routines that under certain circumstances +could report an array was using the unlimited dimension incorrectly. + +- Removed the option to try to bitwise reproduce Lanai results; due to the +number of changes this is no longer possible. + +- Minor bug fixes to the (seldom used) perturb routines in the **WRF** +and **mpas_atm** `model_mod.f90` files. (used to add gaussian noise to a +single model state to generate an ensemble; this is never the recommended +method of starting a new experiment but the code remains for testing +purposes.) + +- Several remaining model-specific `model_mod_check` programs were +removed in favor of a single common program source file. + +- Keep `filter_mod.dopplerfold.f90` in sync with `filter_mod.f90`, +and `assim_tools_mod.pf.f90` in sync with `assim_tools_mod.f90`. + +- Removed makefiles for the obsolete `trans_time` program. + +------------------------------------------------------------------------------ +## Jun 18 2018 :: CAM/CESM 2.0, DART QC 8, closest_member_tool Revision: 12682 + +- Support for **cam-fv** assimilations in the CESM 2.0 release. See + documentation in `models/cam-fv/doc/README_cam-fv` for details. + +- `obs_diag` and matlab scripts updated to report statistics on DART QC 8, + observation failed vertical conversion + +- Updates to fix minor problems with the new WRF scripts + +- Added the `inf_sd_max_change` namelist item to all `input.nml` files for + the enhanced inflation option + +- Revival of the `closest_member_tool`, which now runs in parallel on + all ensemble members at one time. This tool can be used as a template + for any other tools which need to process something for all ensemble + members in parallel. + +- Revival of the `fill_inflation_restart` tool as a Fortran 90 program. + Using `ncap2` is still possible, but if the correct version is not + installed or available this tool can be used. + +- Added more functions to the `netcdf_utilities_mod.f90` + +------------------------------------------------------------------------------ +## Aug 03 2018 :: performance fix for distributed mean Revision: 12758 + +- Important performance fix if model does vertical conversion for localization. + Results were not wrong but performance was poor if `distribute_mean = .true.` + was selected in the `&assim_tools_nml` namelist. + + Now distributing the mean runs in close to the non-distributed time and uses + much less memory for large models. This only impacts models which do a vertical + conversion of either the observation or state vertical coordinate for localization + AND which set `&assim_tools_nml :: distribute_mean = .true.` to use less memory. + + When using a distributed mean `convert_all_obs_verticals_first = .true.` should + be set. If your observations will impact most of the model state, then + `convert_all_state_verticals_first = .true.` can also be set. + +------------------------------------------------------------------------------ +## Apr 30 2019 :: cam-fv refactor, posteriors optional, QC 8 Revision: 13138 + +- The CAM Finite Volume (**cam-fv**) `model_mod.f90` has undergone substantial + refactoring to improve simplicity and remove code for unsupported CAM variants + while also supporting WACCM and WACCM-X. Namelist changes will be required. + +- **cam-fv** setup and scripting support added for CESM 2.1, including advanced + archiving and compression + +- fix for WRF's wind direction vectors when using the Polar Stereographic + map projection. Thanks to *Kevin Manning* for the fix. + +- Add filter namelist option to avoid calling the posterior forward operators + and to not create those copies in the `obs_seq.final` file. + +- Use less memory if writing ensemble member values into the `obs_seq.final` file. + +- added a DART QC of 8 for failed vertical conversions + +- updated Matlab scripts support QC=8 and no posterior in obs sequence files. + +- sampling error correction table now has all ensemble sizes between 3 and 200 + +- `closest_member_tool` can be compiled with other MPI targets + +- `COSMIC_ELECTRON_DENSITY` has been moved from `obs_def_gps_mod.f90` to + `obs_def_upper_atm_mod.f90`, which has new quantities for + `ION_O_MIXING_RATIO` and `ATOMIC_H_MIXING_RATIO` + +- `obs_converters/gps/convert_cosmic_ionosphere.f90` has a test dataset + +- support for NAG compiler + +- fixed Intel compiler bug in `lorenz_96` comparing long integers to integer loop indices + +- `get_maxdist()` now a required routine all location modules + +- Default routines now create a time variable as `time(time)` to allow multiple + files to be concatenated along the unlimited dimension more easily. Also + conforms to the netCDF convention for coordinate dimensions. + +- `obs_impact_tool` handles a continuum of values, not just discrete 0 or 1. + +- `fill_inflation_restart` now produces files with names consistent with filter defaults. + +- expanded functionality in `xyz_location_mod.f90` + +- Removed 'slow' sorting routines from `sort_mod.f90` + +- replacing some repeated native netCDF library calls with routines from + the `netcdf_utilities_mod.f90` + +- Updated dewpoint equation to avoid dividing by zero given a very unlikely + scenario (r12832) + +- More efficient implementation of adaptive inflation + +- *Yongfei Zhang* and *Cecilia Bitz* added improvements to the CICE model and + observation converters and forward operators. These changes also use the + locations of the 'new' glade filesystem. They used CESM tag: cesm2_0_alpha06n + +- Worked with Yongfei Zhang to remove prototype codes and more completely + document observation converters and data sources for cice assimilation. + +- removed `allow_missing_in_clm` flag from the `&assim_tools_nml` namelist in + the CICE work directory. The flag moved to a different namelist and the + CICE model doesn't care about it. + +- increased the maximum number of input files to `obs_diag` from 100 to 10000. + +- Updated the `developer_tests` to include more cases. + +- Updated `oned/obs_diag.f90` to support `obs_seq.out` files. + +- Better error and informational messages in various routines. + +------------------------------------------------------------------------------ +## Nov 20 2019 :: FESOM,NOAH-MP model support, better testing Tag: v9.8.0 + +- first release entirely from GIT + +- fixed bug in `fill_inflation_restart` tool which used the prior inflation mean + and sd for both prior and posterior inflation files. now correctly uses the + posterior mean/sd if requested. + +- fixed a typo in the location test script that prevented it from running + +- additional functionality in the quad interpolation code, now supports grids + which start at 90 (north) and end at -90 (south). + +- if possible, send shorter MPI messages. improves performance on some platforms + and MPI implementations. + +- add explicit call to `initalize_utilities()` where it was missing in a couple of + the WRF utility routines. + +- added an example of how to use a namelist to the `text_to_obs.f90` observation + converter program. + +- Removing the clamping messages in `clamp_variable()` of clamped values + +- changed argument names using reserved keywords. + - `state_vector_io_mod:read_state() 'time' to 'model_time'` + - `random_seq_mod:random_gamma() 'shape' to 'rshape', 'scale' to 'rscale'` + - `random_seq_mod:random_inverse_gamma() 'shape' to 'rshape', 'scale' to 'rscale'` + - `obs_def_mod:init_obs_def() 'kind' to 'obkind', 'time' to 'obtime'` + - `obs_utilities_mod: 'start' to 'varstart', 'count' to 'varcount'` + +- The **FESOM** model is now Manhattan-ready. Thanks to **Ali Aydogdu** + +- The **noah** model is now Manhattan-ready and may be used with NOAH-MP. + +- bugfixed references to the `documentation` directory that was renamed + `docs` to comply with GitHub Pages. + +- improved `test_dart.csh` functionality. + +------------------------------------------------------------------------------ +## MMM DD YYYY :: summary of changes in next Manhattan update Tag: vX.Y.Z + +- Added the MPD observation converter for ABSOLUTE_HUMIDITY observations. + Thanks to Michael Ying for contributing it. + +- Converted CHANGELOG to a markdown document. + +- Converted `observations/obs_converters/observations.html` to `observations/obs_converters/README.md` diff --git a/assimilation_code/modules/observations/obs_sequence_mod.html b/assimilation_code/modules/observations/obs_sequence_mod.html index c6ac1cd1cf..620da62901 100644 --- a/assimilation_code/modules/observations/obs_sequence_mod.html +++ b/assimilation_code/modules/observations/obs_sequence_mod.html @@ -49,7 +49,7 @@

Overview

These routines are commonly used in conversion programs which read observation data from various formats and create a DART observation sequence in memory, and then write it out to a file. See the -observations +observations directory for examples of programs which create and manipulate observations using this routines.

diff --git a/assimilation_code/programs/create_obs_sequence/create_obs_sequence.html b/assimilation_code/programs/create_obs_sequence/create_obs_sequence.html index 5a8ee8b69e..da72ef755c 100644 --- a/assimilation_code/programs/create_obs_sequence/create_obs_sequence.html +++ b/assimilation_code/programs/create_obs_sequence/create_obs_sequence.html @@ -36,7 +36,7 @@

Overview

or shorter sequences of observations (although there is no limit on the number of observations). For creating observation sequence files directly from large, real-world observation datasets, see the -observations directory. +observations directory.

diff --git a/docs/pages/Models.md b/docs/pages/Models.md index a9493e08c4..fe0c9ad590 100644 --- a/docs/pages/Models.md +++ b/docs/pages/Models.md @@ -850,7 +850,7 @@ There are lots of ways to define an observation sequence that DART can use as input for a perfect model experiment. If you have observations in DART format already, you can simply use them. If you have observations in one of the formats already supported by the DART converters -(check [DART/observations/obs_converters/observations.html](obs_converters_observations.html)), +(check [DART/observations/obs_converters/README.md](obs_converters/README.md)), convert it to a DART observation sequence. You may need to use the [obs_sequence_tool](../../assimilation_code/programs/obs_sequence_tool/obs_sequence_tool.html) to combine multiple observation sequence files into observation sequence diff --git a/docs/pages/Observations.md b/docs/pages/Observations.md index e9e8a3956f..8aa167e024 100644 --- a/docs/pages/Observations.md +++ b/docs/pages/Observations.md @@ -415,8 +415,8 @@ the model consistent with those observations - the true state: Real observations come in a mind-boggling diversity of formats. We have converters for many formats in the `DART/observations/obs_converters` -directory. The documentation for that directory is listed in -[observations.html](Manhattan/observations/obs_converters/observations.html). +directory. The documentation for that directory is listed in the +[README.md](../../observations/obs_converters/README.md). The converters are designed to work on one input file format and create (or add to) an output observation sequence. It may be desirable to diff --git a/observations/obs_converters/Ameriflux/level4_to_obs.html b/observations/obs_converters/Ameriflux/level4_to_obs.html index 58050de370..e4737df42c 100644 --- a/observations/obs_converters/Ameriflux/level4_to_obs.html +++ b/observations/obs_converters/Ameriflux/level4_to_obs.html @@ -354,7 +354,7 @@

DECISIONS YOU MIGHT NEED TO MAKE

See the discussion in the -observations introduction +obs_converters/README.md page about what options are available for the things you need to specify. These include setting a time, specifying an expected error, setting a location, and an observation type. diff --git a/observations/obs_converters/MODIS/MOD15A2_to_obs.html b/observations/obs_converters/MODIS/MOD15A2_to_obs.html index 067d578cf7..725490b882 100644 --- a/observations/obs_converters/MODIS/MOD15A2_to_obs.html +++ b/observations/obs_converters/MODIS/MOD15A2_to_obs.html @@ -442,7 +442,7 @@

DECISIONS YOU MIGHT NEED TO MAKE

See the general discussion in the -observations introduction +obs_converters/README.md page about what options are available for the things you need to specify. These include setting a time, specifying an expected error, setting a location, and an observation type. diff --git a/observations/obs_converters/MPD/text_to_obs.f90 b/observations/obs_converters/MPD/MPD_text_to_obs.f90 similarity index 96% rename from observations/obs_converters/MPD/text_to_obs.f90 rename to observations/obs_converters/MPD/MPD_text_to_obs.f90 index e28e9e26fb..6d1888527e 100644 --- a/observations/obs_converters/MPD/text_to_obs.f90 +++ b/observations/obs_converters/MPD/MPD_text_to_obs.f90 @@ -2,11 +2,11 @@ ! by UCAR, "as is", without charge, subject to all terms of use at ! http://www.image.ucar.edu/DAReS/DART/DART_download -program text_to_obs +program MPD_text_to_obs !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ! -! MPD/text_to_obs - a program that only needs minor customization to read +! MPD/MPD_text_to_obs - a program that only needs minor customization to read ! in a text-based dataset - either white-space separated values or ! fixed-width column data. ! @@ -37,7 +37,7 @@ program text_to_obs implicit none -character(len=*), parameter :: source = 'MPD/text_to_obs.f90' +character(len=*), parameter :: source = 'MPD_text_to_obs.f90' character(len=*), parameter :: text_input_file = 'text.txt' character(len=*), parameter :: obs_out_file = 'obs_seq.out' @@ -93,7 +93,7 @@ program text_to_obs ! This is sort of a do-nothing call to initialize the obs_kind_mod module. ! The only real purpose of this call is to make obs_kind_mod print its -! initialization report at the beginning of the output from text_to_obs. +! initialization report at the beginning of the output from MPD_text_to_obs. ! Without this call, the initialization report actually comes AFTER all ! the observations have been processed and right before the output file ! is closed. We do not actually need to know what is in string1. @@ -190,4 +190,4 @@ program text_to_obs ! end of main program call finalize_utilities() -end program text_to_obs +end program MPD_text_to_obs diff --git a/observations/obs_converters/MPD/work/.gitignore b/observations/obs_converters/MPD/work/.gitignore new file mode 100644 index 0000000000..24327b88fb --- /dev/null +++ b/observations/obs_converters/MPD/work/.gitignore @@ -0,0 +1 @@ +MPD_text_to_obs diff --git a/observations/obs_converters/MPD/work/mkmf_text_to_obs b/observations/obs_converters/MPD/work/mkmf_MPD_text_to_obs similarity index 72% rename from observations/obs_converters/MPD/work/mkmf_text_to_obs rename to observations/obs_converters/MPD/work/mkmf_MPD_text_to_obs index c9a748ebff..b883b24200 100755 --- a/observations/obs_converters/MPD/work/mkmf_text_to_obs +++ b/observations/obs_converters/MPD/work/mkmf_MPD_text_to_obs @@ -4,9 +4,9 @@ # by UCAR, "as is", without charge, subject to all terms of use at # http://www.image.ucar.edu/DAReS/DART/DART_download - ../../../../build_templates/mkmf -p text_to_obs \ + ../../../../build_templates/mkmf -p MPD_text_to_obs \ -t ../../../../build_templates/mkmf.template \ - -a "../../../.." path_names_text_to_obs + -a "../../../.." path_names_MPD_text_to_obs exit $status diff --git a/observations/obs_converters/MPD/work/path_names_text_to_obs b/observations/obs_converters/MPD/work/path_names_MPD_text_to_obs similarity index 96% rename from observations/obs_converters/MPD/work/path_names_text_to_obs rename to observations/obs_converters/MPD/work/path_names_MPD_text_to_obs index b551ddca46..b440b5da13 100644 --- a/observations/obs_converters/MPD/work/path_names_text_to_obs +++ b/observations/obs_converters/MPD/work/path_names_MPD_text_to_obs @@ -23,5 +23,5 @@ models/template/model_mod.f90 models/utilities/default_model_mod.f90 observations/forward_operators/obs_def_mod.f90 observations/forward_operators/obs_def_utilities_mod.f90 -observations/obs_converters/MPD/text_to_obs.f90 +observations/obs_converters/MPD/MPD_text_to_obs.f90 observations/obs_converters/utilities/obs_utilities_mod.f90 diff --git a/observations/obs_converters/README b/observations/obs_converters/README deleted file mode 100644 index 84a99f4731..0000000000 --- a/observations/obs_converters/README +++ /dev/null @@ -1,22 +0,0 @@ -# DART software - Copyright UCAR. This open source software is provided -# by UCAR, "as is", without charge, subject to all terms of use at -# http://www.image.ucar.edu/DAReS/DART/DART_download -# -# DART $Id$ - -18 November 2008 - -This directory is a work in progress. There are currently about -10 other observation sources and types which we are in the process -of collecting information and conversion programs for and which will -eventually be added to this directory. In the meantime, if you have -converters for data or interest in something that is not in the -repository, please email the DART group. - -Thanks for your patience, -The DART Development Group - -# -# $URL$ -# $Revision$ -# $Date$ diff --git a/observations/obs_converters/README.md b/observations/obs_converters/README.md new file mode 100644 index 0000000000..a731dfd5c5 --- /dev/null +++ b/observations/obs_converters/README.md @@ -0,0 +1,393 @@ + + +DART Observations +================= + +--- +![DART project logo](https://github.com/NCAR/DART/tree/Manhattan/docs/images/Dartboard7.png){height="70"} +--- + +[OVERVIEW](#Overview) / [DATA SOURCES](#datasourcesandformats) / +[DECISIONS](#Decisions) / [PROGRAMS](#Programs) / [KNOWN BUGS](#knownbugs) / +[FUTURE PLANS](#futureplans) / [TERMS OF USE](#termsofuse) + +Overview +-------- + +Real-world observations of earth-system data come from a variety of +sources, including radiosondes, satellites, ships, aircraft, weather +stations, etc. The files in this *observations* directory can be used to +convert data from a variety of native formats into a common DART +*observation sequence* format. + +Synthetic observations are those not based on an actual instrument +reading of a system, but instead are fabricated to have a known value, +or have values computed by running a model, possibly with a fixed amount +of simulated noise added. These observations can be used for testing, +determining the sensitivity of the model to assimilation, and for +designing new observation systems. The DART system includes several ways +to create synthetic observations. See the [Programs](#Programs) section +below for more details. + +The DART framework enforces a clean separation between observations and +the models they are assimilated into. The same observations can be used +in any model which understands how to generate a value for the requested +type of observation from its state space values. + +In many cases a single, self-contained program can convert directly from +the observation location, time, value, and error into the DART format. +In other cases, especially those linking with a complicated external +library (e.g. BUFR), there is a two-step process with two programs and +an ASCII intermediate file. We are currently leaning towards single-step +conversions but either approach can be used for new programs. + +Frequently the original datasets are in a standard scientific format +like netCDF, HDF, or BUFR, and library routines for those formats can be +used to read in the original observation data. + +The DART software distribution includes Fortran subroutines and +functions to help create a sequence of observations in memory, and then +a call to the DART observation sequence write routine will create an +entire *obs\_seq* file in the correct format. + +The DART system comes with several types of location modules for +computing distances appropriately. Two of the ones most commonly used +are for data in a 1D system and for data in a 3D spherical coordinate +system. All the programs here assume the +*location/threed\_sphere/location\_mod.f90* 3D sphere location module is +being used. + +There are currently some additional observation sources and types which +we are in the process of collecting information and conversion programs +for and which will eventually be added to this directory. In the +meantime, if you have converters for data or interest in something that +is not in the repository, please [email the DART group](mailto:dart@ucar.edu). + +

+ +\[[top](#)\] + +
+ +------------------------------------------------------------------------ + +DATA SOURCES AND FORMATS +------------------------ + +See the various subdirectories here, which generally include information +on where the example data was obtained and in what format it is +distributed. Most data is available for download off the web. The Data +Support Section (DSS) at NCAR has large data repositories, the MADIS +data center distributes observations in NetCDF format, GTS real-time +weather data is available from various sources. For new converters, if +you can find what format the data is distributed in you may be able to +adapt one of the existing converters here for your own use. Formats read +by the existing converters include NetCDF, HDF, little-r, text, +Prepbufr, amongst others. + +**See the [Programs](#Programs) section below for a list of the current +converter programs. It might save you from reinventing the wheel.** + +If you have looked and none of the existing converters are right for +your data, here are some suggestions for where to start creating a new +converter. Create a new subdirectory in the *observations* directory. +Copy with the recursive option (*cp -r*) one of the existing converters +and adapt to your needs. Our suggestions for which converter to start +from depends on the format of your input observations to be converted. +If your input data format is: + + + + + + + + + + + + + + + + + + + + + + + + + + +
netCDFStart with the MADIS converters, and in particular try the + convert_madis_profiler.f90 file because it is the most + straightforward. Another good option is SST/oi_sst_to_obs.f90
Comma separated textStart with the Ameriflux converter.
Generic textStart with the text converter.
HDF-EOSStart with the AIRS converter.
BUFR or prepBUFRStart with the NCEP converter.
Dense data, like Satellite swathsStart with the tpw converter, which includes code that + averages the raw data in space and time.
Ray-path integrated dataStart with the GPS converter, which includes code that +traces a path and integrates values along the ray.
World Ocean Database packed ASCIIStart with the WOD converter.
+ + + +
+ +\[[top](#)\] + +
+ +------------------------------------------------------------------------ + +DECISIONS YOU MIGHT NEED TO MAKE +-------------------------------- + +#### Time + +Time enters into the assimilation system in 3 places: the timestamp of +the state vector data (the current model time when this data was +produced), the time of each observation, and the minimum time period the +model should be called to advance (the assimilation window size). The +internal timestepping of the model is unrelated to any of these times +and is outside the scope of the assimilation system. + +The basic time type in DART is a pair of integers; one for the day +number and one for the number of seconds. Generally the low order +models, which aren't direct geophysical models, use time directly as a +sequence of days starting at 0 and incrementing in any appropriate +number of seconds or days. The observations assimilated into these +systems do not need to use a calendar. + +Observations of a real-world system usually are distributed with a +year/month/day, hour/min/seconds timestamp. There are routines in DART +to convert back and forth between the (day-number/seconds) format and a +variety of (year/month/day) calendars. See [the time manager +documentation](../../assimilation_code/modules/utilities/time_manager_mod.html#time_type) +for more details on how DART stores time information and the types of +available calendars. Some climate models which do long runs (100s or +1000s of years) use a modified calendar for simplicity in computation, +e.g. months which always have 30 days, or no leap years. When trying to +assimilate real observations into these models there may be calendar +issues to solve. + +The smallest resolvable unit of time in DART is a second. To model a +system which operates on sub-second time scales the time can be scaled +up by some factor. As long as the observation time, the state data time, +and the minimum model advance time are expressed in the same scaled time +units, there is no problem. + +#### Error + +Observations must specify an associated expected error. Each individual +observation stores its own error value, so it can be a constant value +for all observations of that type or it can vary by location, by height, +by magnitude of the observed value, etc. This value is the expected +instrument error plus the representativeness error of the model. The +model error includes deficiencies in the equations representing the +processes of the system as well as errors introduced by representing a +continuous system as a series of discrete points. While the instrument +error and the representativeness error could be specified separately, +they each have the same impact on the assimilation and can be difficult +to determine with any real accuracy. For simplicity, in DART (and most +current assimilation software) they are combined and specified as a +single value. + +The instrument error is generally supplied by the instrument maker. +Sadly, it is frequently surprisingly difficult to find these values. For +the representativeness error, a set of artificial observations could be +generated with the +[perfect\_model\_obs](../../assimilation_code/programs/perfect_model_obs/perfect_model_obs.html) +program and an assimilation experiment could be run to generate an +estimate of the error in the model. In practice however most people make +an educated guess on the values of the error and then start with a +larger than expected value and decrease it based on the results of +running some test assimilations. For these tests the namelist for the +[outlier +threshold](../../assimilation_code/programs/filter/filter.html#Namelist) +should be disabled by setting it to -1 (the default value is 3). This +value controls whether the observation is rejected because the observed +value is too far from the ensemble mean. + +If the diagnostics show that the difference between the mean of the +forward operators and the observed value is consistently smaller than +the specified observation error, then the error is probably too large. A +too-large error reduces the impact of an observation on the state. If +the specified observation error is too small it is likely the +observation will be rejected when the outlier threshold is enabled, and +the observation will not be assimilated. It is important to look at the +output observation sequence files after an assimilation to see how many +observations were assimilated or rejected, and also at the RMSE ([root +mean squared error](http://www.wikipedia.org/wiki/RMSE)) versus the +total spread. DART includes Matlab diagnostic routines to create these +types of plots. The observation RMSE and total spread should be roughly +commensurate. The total spread includes contributions from both the +ensemble variance and the observational error variance, so it can be +adjusted by changing the error values on the incoming observations. +There are other ways to adjust the ensemble spread, including +[inflation](../../assimilation_code/programs/filter/filter.html#Inflation), +so the observation error is not the only factor to consider. + +One last recommendation: if possible, the Prior forward operator values +should be compared against the observations after several assimilation +cycles. If you plot results using the Posterior values it is always +possible for the assimilation to overfit the observations and look good +on the diagnostic plots. But the actual test is to then advance the +model and look at how the forecast of the state compares to the +observations. + +#### Types + +All observations have to have a specific 'type'. There are namelist +controls to turn on and off the assimilation of observations at run-time +by type, or to only evaluate the forward operator for an observation but +have no impact on the state. Several of the diagnostics also group +observations by type to give aggregate statistics after an assimilation. +Generally types are based on both the observing platform or instrument +as well as the kind of observation, e.g. RADIOSONDE\_TEMPERATURE, +ARGO\_SALINITY, etc. Each type is associated with a single underlying +generic 'kind', which controls what forward operator code is called +inside the model, e.g. QTY\_TEMPERATURE, QTY\_DENSITY, etc. + +See [here](../forward_operators/obs_def_mod.html) for more details on +how to use and add new DART types. The DART obs\_kind\_mod.f90 defines a +list of already defined observation kinds, and users can either use +existing observation types in 'obs\_def\_xxx\_mod.f90' files, or define +their own. + +#### Locations + +The two most common choices for specifying the location of an +observation are the +[threed\_sphere](../../assimilation_code/location/threed_sphere/location_mod.html) +and the [oned](../../assimilation_code/location/oned/location_mod.html) +locations. For observations of a real-world system, the 3D Sphere is +generally the best choice. For low-order, 1D models, the 1D locations +are the most commonly used. The observation locations need to match the +type of locations used in the model. + +[]{#Programs} + +
+ +\[[top](#)\] + +
+ +------------------------------------------------------------------------ + +PROGRAMS +-------- + +The *DART/observations/obs\_converters* directory contains a variety of +converter programs to read various external formats and convert the +observations into the format required by DART. + +The current list of converters include: + +- [AIRS](AIRS/AIRS.html) +- [Aviso+/CMEMS](AVISO/AVISO.html) +- [Ameriflux](Ameriflux/level4_to_obs.html) +- [COSMOS](COSMOS/COSMOS_to_obs.html) +- [DWL](DWL/dwl_to_obs.html) +- [GPSPW](GPSPW/README) +- [GSI2DART](GSI2DART/README) +- [GTSPP](GTSPP/GTSPP.html) +- [MADIS](MADIS/MADIS.html) +- [MIDAS](MIDAS/MIDAS_to_obs.html) +- [MODIS](MODIS/MOD15A2_to_obs.htm) +- [MPD](MPD/README.md) +- [NCEP (prepbufr->ascii)](NCEP/prep_bufr/prep_bufr.html) +- [NCEP (ascii->obs\_seq)](NCEP/ascii_to_obs/create_real_obs.html) +- [ROMS](ROMS/ROMS.htm) +- [SSEC](SSEC/SSEC.html) +- [SST](SST/SST.html) +- [SSUSI](SSUSI/convert_f16_edr_dsk.html) +- [WOD](WOD/WOD.html) +- [cice](cice/cice_to_obs.html) +- [gnd\_gps\_vtec](gnd_gps_vtec/README) +- [GPS](gps/gps.html) +- [ok\_mesonet](ok_mesonet/ok_mesonet.html) +- [QuikSCAT](quikscat/QuikSCAT.html) +- [Radar](radar/radar.html) +- [snow](snow/snow_to_obs.html) +- [Text](text/text_to_obs.html) +- [tpw](tpw/tpw.html) +- [Tropical Cyclones](tropical_cyclone/tc_to_obs.html) +- [Var (little-r)](var/littler_tf_dart.html) +- [Var (radar)](var/rad_3dvar_to_dart.html) + +There are also a couple utilities of note: + +- [even\_sphere](even_sphere/README) - a utility for generating + evenly-spaced observation locations that can then be used in a + perfect model experiment. +- [obs\_error](obs_error/README) - modules that specify observation + errors based on what is used by ECMWF and NCEP + +In addition the following external program produces DART observation +sequence files: + +- [Observation Processing And Wind Synthesis + (OPAWS)](http://code.google.com/p/opaws/): OPAWS can process NCAR + Dorade (sweep) and NCAR EOL Foray (netcdf) radar data. It analyzes + (grids) data in either two-dimensions (on the conical surface of + each sweep) or three-dimensions (Cartesian). Analyses are output in + netcdf, Vis5d, and/or DART (Data Assimilation Research Testbed) + formats. + +For generating synthetic observations, see the +[create\_obs\_sequence](../../assimilation_code/programs/create_obs_sequence/create_obs_sequence.html) +program documentation. You can also generate observation files based on +text input. See the [text\_to\_obs](text/text_to_obs.html) program +documentation. Or for simulating a large complex observing system, you +can use the DART library routines in a Fortran program to compute the +observation information and have the DART routines write the output +file. + +See the +[perfect\_model](../../assimilation_code/programs/perfect_model_obs/perfect_model_obs.html) +program documentation on how to run a model with a set of observations +that have only locations, types, and times, and have the forward +operators compute the observation values. + + +
+ +\[[top](#)\] + +
+ +------------------------------------------------------------------------ + +KNOWN BUGS +---------- + +  + +
+ +\[[top](#)\] + +
+ +------------------------------------------------------------------------ + +FUTURE PLANS +------------ + +Contact the [DART development group](mailto:dart@ucar.edu) if you have +observations in a different format that you want to convert. We can give +you advice and pointers on how to approach writing the code. + +
+ +\[[top](#)\] + +
+ +------------------------------------------------------------------------ + +Terms of Use +------------ + +DART software - Copyright UCAR. This open source software is provided by +UCAR, "as is", without charge, subject to all terms of use at + diff --git a/observations/obs_converters/SST/SST.html b/observations/obs_converters/SST/SST.html index 29d5d66a5c..69972901de 100644 --- a/observations/obs_converters/SST/SST.html +++ b/observations/obs_converters/SST/SST.html @@ -391,7 +391,7 @@

DECISIONS YOU MIGHT NEED TO MAKE

See the general discussion in the -observations introduction +obs_converters/README.md page about what options are available for the things you need to specify. These include setting a time, specifying an expected error, setting a location, and an observation type. diff --git a/observations/obs_converters/text/text_to_obs.html b/observations/obs_converters/text/text_to_obs.html index e34bb58bde..417b2ad8c0 100644 --- a/observations/obs_converters/text/text_to_obs.html +++ b/observations/obs_converters/text/text_to_obs.html @@ -133,7 +133,7 @@

DECISIONS YOU MIGHT NEED TO MAKE

See the discussion in the -observations introduction +obs_converters/README.md page about what options are available for the things you need to specify. These include setting a time, specifying an expected error, setting a location, and an observation type. From ccbef93d833aa77585b00fe558d09b54b8373056 Mon Sep 17 00:00:00 2001 From: Tim Hoar Date: Wed, 15 Apr 2020 20:47:08 -0600 Subject: [PATCH 09/16] new location of DART logo --- observations/obs_converters/README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/observations/obs_converters/README.md b/observations/obs_converters/README.md index a731dfd5c5..724cca2dea 100644 --- a/observations/obs_converters/README.md +++ b/observations/obs_converters/README.md @@ -4,7 +4,7 @@ DART Observations ================= --- -![DART project logo](https://github.com/NCAR/DART/tree/Manhattan/docs/images/Dartboard7.png){height="70"} +![DART project logo](https://github.com/NCAR/DART/blob/Manhattan/docs/images/Dartboard7.png) --- [OVERVIEW](#Overview) / [DATA SOURCES](#datasourcesandformats) / @@ -263,8 +263,6 @@ generally the best choice. For low-order, 1D models, the 1D locations are the most commonly used. The observation locations need to match the type of locations used in the model. -[]{#Programs} -

\[[top](#)\] From a861fea237e371e6e27ca15632a7f1c58d1e8cdc Mon Sep 17 00:00:00 2001 From: Tim Hoar Date: Wed, 22 Apr 2020 10:06:59 -0600 Subject: [PATCH 10/16] Initial documentaion for absolute humidity from MPD --- CHANGELOG.md | 3 ++ COPYRIGHT.md | 2 +- observations/obs_converters/MPD/README.md | 45 +++++++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 observations/obs_converters/MPD/README.md diff --git a/CHANGELOG.md b/CHANGELOG.md index e7fabf2316..07797c5e2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +![DARTlogo](docs/images/Dartboard7.png) + DART software - Copyright UCAR. This open source software is provided by UCAR, "as is", without charge, subject to all terms of use at http://www.image.ucar.edu/DAReS/DART/DART_download @@ -9,6 +11,7 @@ It is not intended to document every change, but instead is intended to inform people what features are now available or have been removed. Detailed changes are always available through the version control framework. +The Revision numbers can be related to git hashes by 'git log ## SEE THE BOTTOM OF THIS FILE FOR THE MOST RECENT CHANGES! ---- diff --git a/COPYRIGHT.md b/COPYRIGHT.md index 2d6056b9d2..477ac75305 100644 --- a/COPYRIGHT.md +++ b/COPYRIGHT.md @@ -1,5 +1,5 @@ -![DARTlogo](documentation/images/Dartboard7.png "DART Logo") +![DARTlogo](docs/images/Dartboard7.png "DART Logo") Copyright 2019 University Corporation for Atmospheric Research diff --git a/observations/obs_converters/MPD/README.md b/observations/obs_converters/MPD/README.md new file mode 100644 index 0000000000..1d2c17891a --- /dev/null +++ b/observations/obs_converters/MPD/README.md @@ -0,0 +1,45 @@ + +![DARTlogo](docs/images/Dartboard7.png) + +The **M**icro**P**ulse **D**ifferential Absorption Lidar (**MPD**) data +were collected during field campaigns and testing periods by the +Earth Observing Laboratory (EOL). + +The differential absorption lidar (DIAL) technique uses two separate +laser wavelengths: an absorbing wavelength (online) and a non-absorbing +wavelength (offline). The ratio of the range-resolved backscattered +signals between the online and offline wavelengths is proportional to +the amount of water vapor in the atmosphere, which allows the retrieval +of absolute humidity profiles above the lidar site. + +**THIS** + +This observation converter takes absolute humidity (g/m3) profiles +retrieved from the MPD data and convert to format used by DART. +A sample netCDF data for the retrieved MPD absolute humidity is provided +in `data/wv_mpd01.190614.dafinal.Python.nc`. + +The `obs_converter/MPD/work/convert_to_text.py` script reads the netCDF +files from each MPD site and combine them into text files, one for each +date/time. An example text file is provided in at `data/text.txt`. +The `obs_converter/MPD/work/mpd_text_to_obs` program translates the text +files to DART `obs_seq.out` format. + +**OR** + +This observation converter takes absolute humidity (g/m3) profiles +retrieved from the MPD data and convert to format used by DART. +The `obs_converter/MPD/work/convert_to_text.py` script reads the netCDF +files from each MPD site and combine them into text files, one for each +date/time. The `obs_converter/MPD/work/MPD_text_to_obs` program +translates the text files to DART `obs_seq.out` format. + +Test data for a single site and an example output can be downloaded from +https://www.image.ucar.edu/pub/DART/TestData/MPD.tar + +**THAT** + +For more details of the retrieval and quality control process, +and inquire about data availability for your research project, +please contact Tammy Weckwerth at EOL, NCAR. + From b38eca74aec1194a0bb28ecf7207241b1f5e8eb4 Mon Sep 17 00:00:00 2001 From: Tim Hoar Date: Wed, 22 Apr 2020 10:57:47 -0600 Subject: [PATCH 11/16] Adding content for MPD readme. Other files have formatting changes, no changed content. --- CHANGELOG.md | 6 ++++++ COPYRIGHT.md | 4 ++++ README.md | 4 ++++ docs/pages/Presentations.md | 4 ++++ docs/pages/Publications.md | 4 ++++ models/FESOM/README.md | 5 +++++ models/FESOM/diagnostics/script/README.md | 5 +++++ observations/obs_converters/MPD/README.md | 4 ++++ observations/obs_converters/README.md | 5 +++++ 9 files changed, 41 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07797c5e2e..a92f12e2c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +--- +title: Changelog +layout: default +--- + + ![DARTlogo](docs/images/Dartboard7.png) DART software - Copyright UCAR. This open source software is provided diff --git a/COPYRIGHT.md b/COPYRIGHT.md index 477ac75305..80f59bde56 100644 --- a/COPYRIGHT.md +++ b/COPYRIGHT.md @@ -1,3 +1,7 @@ +--- +title: Apache License +layout: default +--- ![DARTlogo](docs/images/Dartboard7.png "DART Logo") diff --git a/README.md b/README.md index 90f1a9ab94..a6ebcb3e52 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +--- +title: Welcome to DART +layout: default +--- ![DARTlogo](docs/images/Dartboard7.png) diff --git a/docs/pages/Presentations.md b/docs/pages/Presentations.md index 85735c50aa..c2c1b16ac6 100644 --- a/docs/pages/Presentations.md +++ b/docs/pages/Presentations.md @@ -1,3 +1,7 @@ +--- +title: DART Presentations +layout: default +--- # (Some) Presentations featuring DART: The Data Assimilation Research Testbed diff --git a/docs/pages/Publications.md b/docs/pages/Publications.md index 40b4d0716e..7e2dc9aa6e 100644 --- a/docs/pages/Publications.md +++ b/docs/pages/Publications.md @@ -1,3 +1,7 @@ +--- +title: DART Publications +layout: default +--- # Publications using DART diff --git a/models/FESOM/README.md b/models/FESOM/README.md index 43a6b6fa31..480b6eac8b 100644 --- a/models/FESOM/README.md +++ b/models/FESOM/README.md @@ -1,3 +1,8 @@ +--- +title: Finite Element Sea-ice Ocean Model +layout: default +--- + ## **Finite Element Sea-ice Ocean Model (FESOM)** FESOM is an unstructured mesh global ocean model using finite element methods to solve the diff --git a/models/FESOM/diagnostics/script/README.md b/models/FESOM/diagnostics/script/README.md index f8f5bb1581..ab5ff028d8 100644 --- a/models/FESOM/diagnostics/script/README.md +++ b/models/FESOM/diagnostics/script/README.md @@ -1,3 +1,8 @@ +--- +title: FESOM scripting +layout: default +--- + ### Further description of the scripts Scripts call a fortran tool to post-process the FESOM and DART outputs. They include a **tool** variable diff --git a/observations/obs_converters/MPD/README.md b/observations/obs_converters/MPD/README.md index 1d2c17891a..34e157c172 100644 --- a/observations/obs_converters/MPD/README.md +++ b/observations/obs_converters/MPD/README.md @@ -1,3 +1,7 @@ +--- +title: MPD converter +layout: default +--- ![DARTlogo](docs/images/Dartboard7.png) diff --git a/observations/obs_converters/README.md b/observations/obs_converters/README.md index 724cca2dea..7a1cef89f4 100644 --- a/observations/obs_converters/README.md +++ b/observations/obs_converters/README.md @@ -1,3 +1,8 @@ +--- +title: DART Observations +layout: default +--- + DART Observations From e4ae2df78b8a39597c5ea364e99905b09f54a519 Mon Sep 17 00:00:00 2001 From: Tim Hoar Date: Wed, 22 Apr 2020 11:31:25 -0600 Subject: [PATCH 12/16] Removing unwanted gh-pages baggage title/style keys The models/README.md is really out-of-date --- CHANGELOG.md | 7 +-- COPYRIGHT.md | 4 -- README.md | 4 -- docs/pages/Presentations.md | 4 -- docs/pages/Publications.md | 4 -- models/FESOM/README.md | 6 +- models/FESOM/diagnostics/script/README.md | 6 +- models/README.md | 69 ++++++++++++----------- observations/obs_converters/MPD/README.md | 6 +- observations/obs_converters/README.md | 6 +- 10 files changed, 44 insertions(+), 72 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a92f12e2c8..03f752a725 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,7 @@ ---- -title: Changelog -layout: default ---- +![DARTlogo](https://github.com/NCAR/DART/blob/Manhattan/docs/images/Dartboard7.png) -![DARTlogo](docs/images/Dartboard7.png) +---- DART software - Copyright UCAR. This open source software is provided by UCAR, "as is", without charge, subject to all terms of use at diff --git a/COPYRIGHT.md b/COPYRIGHT.md index 80f59bde56..477ac75305 100644 --- a/COPYRIGHT.md +++ b/COPYRIGHT.md @@ -1,7 +1,3 @@ ---- -title: Apache License -layout: default ---- ![DARTlogo](docs/images/Dartboard7.png "DART Logo") diff --git a/README.md b/README.md index a6ebcb3e52..90f1a9ab94 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,3 @@ ---- -title: Welcome to DART -layout: default ---- ![DARTlogo](docs/images/Dartboard7.png) diff --git a/docs/pages/Presentations.md b/docs/pages/Presentations.md index c2c1b16ac6..85735c50aa 100644 --- a/docs/pages/Presentations.md +++ b/docs/pages/Presentations.md @@ -1,7 +1,3 @@ ---- -title: DART Presentations -layout: default ---- # (Some) Presentations featuring DART: The Data Assimilation Research Testbed diff --git a/docs/pages/Publications.md b/docs/pages/Publications.md index 7e2dc9aa6e..40b4d0716e 100644 --- a/docs/pages/Publications.md +++ b/docs/pages/Publications.md @@ -1,7 +1,3 @@ ---- -title: DART Publications -layout: default ---- # Publications using DART diff --git a/models/FESOM/README.md b/models/FESOM/README.md index 480b6eac8b..85d03c8936 100644 --- a/models/FESOM/README.md +++ b/models/FESOM/README.md @@ -1,7 +1,5 @@ ---- -title: Finite Element Sea-ice Ocean Model -layout: default ---- + +![DARTlogo](docs/images/Dartboard7.png) ## **Finite Element Sea-ice Ocean Model (FESOM)** diff --git a/models/FESOM/diagnostics/script/README.md b/models/FESOM/diagnostics/script/README.md index ab5ff028d8..5f026dc738 100644 --- a/models/FESOM/diagnostics/script/README.md +++ b/models/FESOM/diagnostics/script/README.md @@ -1,7 +1,5 @@ ---- -title: FESOM scripting -layout: default ---- + +![DARTlogo](docs/images/Dartboard7.png) ### Further description of the scripts diff --git a/models/README.md b/models/README.md index 383517f868..0bad32f8ce 100644 --- a/models/README.md +++ b/models/README.md @@ -1,7 +1,5 @@ ---- -title: Adding a new model to DART -layout: default ---- + +![DARTlogo](docs/images/Dartboard7.png) ## Hints for porting a new model to DART: @@ -23,6 +21,7 @@ to use the name of the directory for your model. try ./quickbuild.csh and everything should compile at this point. the required subroutines are these: +``` public :: get_model_size, & adv_1step, & get_state_meta_data, & @@ -39,29 +38,30 @@ public :: get_model_size, & get_close_obs_init, & get_close_obs, & ens_mean_for_model +``` in addition, model_mod can contain subroutines that are used for other utility programs and we recommend at least the following -routines be added to model_mod.f90: - +routines be added to `model_mod.f90`: +``` public :: model_file_to_dart_vector, & ! converter dart_vector_to_model_file, & ! converter get_gridsize, & ! called by everyone get_model_filename, & ! called by both (set_model_filename?) get_state_time, & ! model_to_sv, static_init_model set_state_time !(?) ! sv_to_model, trans_time - +``` edit the model mod and fill in the routines in this order: -1. static_init_model() - make it read in the grid information +1. `static_init_model()` - make it read in the grid information and the number of variables that will be in the state vector (fill in the progvar derived type). fill in the model_size - variable. as part of this work, fill in the get_gridsize() + variable. as part of this work, fill in the `get_gridsize()` code. - after number 1 is done, get_model_size() and - get_model_time_step() from the template should be ok as-is. + after number 1 is done, `get_model_size()` and + `get_model_time_step()` from the template should be ok as-is. 2. model_file_to_dart_vector() - given a model data file, read in the fields and put them into the 1D DART state vector. make @@ -69,18 +69,18 @@ edit the model mod and fill in the routines in this order: 3. dart_vector_to_model_file() - do the reverse of the previous step. -4. get_state_meta_data() - given an index number into the state vector +4. `get_state_meta_data()` - given an index number into the state vector return the location and type. the code which loops over the progvar should already do the type, but code to compute what lon, lat, and vertical (for a 3d model) or x location (1d) corresponds to this item must be written. -5. model_interpolate() - given a location (lon/lat/vert in 3d, x in 1d) +5. `model_interpolate()` - given a location (lon/lat/vert in 3d, x in 1d) and a state QTY_xxx kind, return the interpolated value the field has at that location. this is probably one of the routines that will take the most code to write. -6. nc_write_model_atts(), nc_write_model_vars() - when filter runs +6. `nc_write_model_atts(), nc_write_model_vars()` - when `filter` runs it calls these routines to output model data into a netcdf diagnostic file which is unrelated to the model data files. it is possible to have the ensemble data just be dumped as a single 1D vector but @@ -91,47 +91,55 @@ edit the model mod and fill in the routines in this order: then optionally some or all of the individual ensemble members. for now, ignore these routines: +``` get_close_maxdist_init() get_close_obs_init() get_close_obs() ens_mean_for_model() end_model() +``` if you have data in a dart initial condition/restart file, then you can ignore these routines: +``` init_time() init_conditions() +``` otherwise, have them return an initial time and an initial default ensemble state. if your model is NOT subroutine callable, you can ignore this routine: +``` adv_1step() +``` otherwise have it call the interface to your model and add the files -necessary to build your model to all the work/path_names_* files. +necessary to build your model to all the `work/path_names_*` files. add the model source to a src/ directory. if you want to let filter add gaussian noise to a single state vector to generate an ensemble, you can ignore this routine +``` pert_model_state() +``` otherwise fill in code that does whatever perturbation makes sense to have an initial ensemble of states. in some cases that means adding a different range of values to each different field in the state vector. at this point you should have enough code to test and run simple -experiments. the 'model_mod_check' utility program can be used +experiments. the `model_mod_check` utility program can be used during this process to check your implementation. the general flow is: - 1) ./model_to_dart - read model data and convert it into a dart state vector file - 2) ./create_obs_sequence - make a file with a single observation in it - 3) ./perfect_model_obs - should interpolate a value for the obs - 4) ./dart_to_model - convert the dart vector back into a model data file + 1) `./model_to_dart` - read model data and convert it into a dart state vector file + 2) `./create_obs_sequence` - make a file with a single observation in it + 3) `./perfect_model_obs` - should interpolate a value for the obs + 4) `./dart_to_model` - convert the dart vector back into a model data file 5) generate an ensemble of states, or set 'start_from_restart' to .false. - 6) run ./filter with the single observation + 6) run `./filter` with the single observation 7) look at the preassim.nc and analysis.nc files diff them with ncdiff: ncdiff analysis.nc preassim.nc Innov.nc plot it, with ncview if possible: ncview Innov.nc @@ -140,7 +148,7 @@ the general flow is: more details on each of these 7 steps follows. -1) model_to_dart +### 1) model_to_dart this program needs to read the output file from the model, whatever format that is (many of our supported models use netcdf). it needs to create a 1d array of values in whatever order it chooses. @@ -150,7 +158,7 @@ that corresponds so, so the mapping from 2d or 3d array to this 1d array has to be kept track of in the model_mod code so it can be inverted on demand. -2) create_obs_sequence +### 2) create_obs_sequence you can make a synthetic observation (or a series of them) with this interactive program and use them for testing. before running, make sure the observation types you want to use are in the input.nml file in the @@ -175,7 +183,7 @@ a single synthetic obs, this will do. for an output filename, it suggests 'seq_def.out' but in this case, tell it 'obs_seq.in'. -3) perfect_model_obs +### 3) perfect_model_obs if you have run the model_to_dart and created a state vector, make sure the name matches the input name in the input.nml file, the &perfect_model_obs_nml namelist. make sure the input obs_sequence is still set to 'obs_seq.in'. @@ -186,7 +194,7 @@ in the input.nml file. the sequence files will be short and in ascii. you can check to see what the interpolated value is. if it's right, yay. if not, debug the interpolation code in the model_mod.f90 file. -4) dart_to_model +### 4) dart_to_model if you have run perfect_model_obs, you have not changed the dart state vector in any way. however, it's a good test to make a copy of the model input file, then run 'model_to_dart' and then 'dart_to_model' @@ -194,7 +202,7 @@ the model input file, then run 'model_to_dart' and then 'dart_to_model' &dart_to_model_nml namelist) and you should get identical values back in the model input file as you started with. -5) running from a single input state +### 5) running from a single input state in the &filter_nml namelist, set 'start_from_restart' to .false. this tells filter that you have not generated N initial conditions, that you are only going to supply one and it needs to perturb that @@ -208,7 +216,7 @@ you may need to set the &ensemble_manager_nml : perturbation_amplitude down to something smaller than 0.2 for these tests - 0.00001 is a good first guess for adding small perturbations to a state. -6) filter +### 6) filter set the ens_size to something small for testing - between 4 and 10 is usually a good range. make sure your observation type is in the 'assimilate_these_obs_types' list and not in the evaluate list. @@ -216,7 +224,7 @@ run filter. find bugs and fix them until the output 'obs_seq.final' seems to have reasonable values. running filter will generate two netcdf diagnostic files: preassim.nc and analysis.nc -7) diagnostics +### 7) diagnostics run 'ncdiff analysis.nc preassim.nc differences.nc' and use your favorite netcdf plotting tool to see if there are any differences between the 2 files. for modules using a regular lat/lon grid 'ncview' @@ -233,8 +241,3 @@ code. if it doesn't have a reasonable value, look at your model_interpolate() c there's lots more to say about this, but this is a quick pointer to how to get started. - -# -# $URL$ -# $Revision$ -# $Date$ diff --git a/observations/obs_converters/MPD/README.md b/observations/obs_converters/MPD/README.md index 34e157c172..dabad08d13 100644 --- a/observations/obs_converters/MPD/README.md +++ b/observations/obs_converters/MPD/README.md @@ -1,9 +1,5 @@ ---- -title: MPD converter -layout: default ---- -![DARTlogo](docs/images/Dartboard7.png) +![DARTlogo](https://github.com/NCAR/DART/blob/Manhattan/docs/images/Dartboard7.png) The **M**icro**P**ulse **D**ifferential Absorption Lidar (**MPD**) data were collected during field campaigns and testing periods by the diff --git a/observations/obs_converters/README.md b/observations/obs_converters/README.md index 7a1cef89f4..6bd5004af6 100644 --- a/observations/obs_converters/README.md +++ b/observations/obs_converters/README.md @@ -1,7 +1,3 @@ ---- -title: DART Observations -layout: default ---- @@ -9,7 +5,7 @@ DART Observations ================= --- -![DART project logo](https://github.com/NCAR/DART/blob/Manhattan/docs/images/Dartboard7.png) +![DARTlogo](https://github.com/NCAR/DART/blob/Manhattan/docs/images/Dartboard7.png) --- [OVERVIEW](#Overview) / [DATA SOURCES](#datasourcesandformats) / From c0261e48c26ead8ad866be579da4ff1053442754 Mon Sep 17 00:00:00 2001 From: Tim Hoar Date: Wed, 22 Apr 2020 11:46:36 -0600 Subject: [PATCH 13/16] using full path to the DART logo --- CHANGELOG.md | 9 +++++++-- models/FESOM/README.md | 4 +++- models/FESOM/diagnostics/script/README.md | 4 +++- models/README.md | 6 +++++- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03f752a725..64ffdde4da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ -![DARTlogo](https://github.com/NCAR/DART/blob/Manhattan/docs/images/Dartboard7.png) +![DARTlogo](docs/images/Dartboard7.png) ---- @@ -14,7 +14,12 @@ It is not intended to document every change, but instead is intended to inform people what features are now available or have been removed. Detailed changes are always available through the version control framework. -The Revision numbers can be related to git hashes by 'git log +The Revision numbers can be related to git hashes by searching the output of +`git log` +``` +0[1011] machine:dartGIT % git log > full_git_log.txt +``` + ## SEE THE BOTTOM OF THIS FILE FOR THE MOST RECENT CHANGES! ---- diff --git a/models/FESOM/README.md b/models/FESOM/README.md index 85d03c8936..e112d7a54a 100644 --- a/models/FESOM/README.md +++ b/models/FESOM/README.md @@ -1,5 +1,7 @@ -![DARTlogo](docs/images/Dartboard7.png) + + +![DARTlogo](https://github.com/NCAR/DART/blob/Manhattan/docs/images/Dartboard7.png) ## **Finite Element Sea-ice Ocean Model (FESOM)** diff --git a/models/FESOM/diagnostics/script/README.md b/models/FESOM/diagnostics/script/README.md index 5f026dc738..c95d39efd4 100644 --- a/models/FESOM/diagnostics/script/README.md +++ b/models/FESOM/diagnostics/script/README.md @@ -1,5 +1,7 @@ -![DARTlogo](docs/images/Dartboard7.png) + + +![DARTlogo](https://github.com/NCAR/DART/blob/Manhattan/docs/images/Dartboard7.png) ### Further description of the scripts diff --git a/models/README.md b/models/README.md index 0bad32f8ce..74396d2615 100644 --- a/models/README.md +++ b/models/README.md @@ -1,5 +1,9 @@ -![DARTlogo](docs/images/Dartboard7.png) + + +![DARTlogo](https://github.com/NCAR/DART/blob/Manhattan/docs/images/Dartboard7.png) + +## This document is really quite out-of-date. TJH 22 Apr 2020 ## Hints for porting a new model to DART: From d3921f9d432cc8433b5b0baeaf7db0e18ab8bf21 Mon Sep 17 00:00:00 2001 From: Tim Hoar Date: Wed, 22 Apr 2020 17:17:35 -0600 Subject: [PATCH 14/16] The example tar file is now in place. --- observations/obs_converters/MPD/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/observations/obs_converters/MPD/README.md b/observations/obs_converters/MPD/README.md index dabad08d13..96d49ec7b6 100644 --- a/observations/obs_converters/MPD/README.md +++ b/observations/obs_converters/MPD/README.md @@ -35,7 +35,7 @@ date/time. The `obs_converter/MPD/work/MPD_text_to_obs` program translates the text files to DART `obs_seq.out` format. Test data for a single site and an example output can be downloaded from -https://www.image.ucar.edu/pub/DART/TestData/MPD.tar +https://www.image.ucar.edu/pub/DART/MPD.tar.gz **THAT** From 941014da7733eb430fbe24e58364395d8e7e0a79 Mon Sep 17 00:00:00 2001 From: Tim Hoar Date: Tue, 28 Apr 2020 13:46:30 -0600 Subject: [PATCH 15/16] reformatted to satisfy pylint --- .../MPD/work/convert_to_text.py | 60 +++++++++---------- 1 file changed, 29 insertions(+), 31 deletions(-) mode change 100755 => 100644 observations/obs_converters/MPD/work/convert_to_text.py diff --git a/observations/obs_converters/MPD/work/convert_to_text.py b/observations/obs_converters/MPD/work/convert_to_text.py old mode 100755 new mode 100644 index f241b2d2d3..95ba245789 --- a/observations/obs_converters/MPD/work/convert_to_text.py +++ b/observations/obs_converters/MPD/work/convert_to_text.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 -# # DART software - Copyright UCAR. This open source software is provided # by UCAR, "as is", without charge, subject to all terms of use at # http://www.image.ucar.edu/DAReS/DART/DART_download @@ -7,36 +5,36 @@ import numpy as np from netCDF4 import Dataset -ccyy = 2019 ##date of data file -mm = 6 -dd = 14 -n_sites = 5 ##number of MPD sites -obs_err = 0.001 ###set observation error 1g/m3 +CCYY = 2019 ##date of data file +MM = 6 +DD = 14 +NUM_SITES = 5 ##number of MPD sites +OBS_ERR = 0.001 ###set observation error 1g/m3 -for site in range(1, n_sites+1): - f = Dataset('wv_mpd0{}.{:02d}{:02d}{:02d}.dafinal.Python.nc'.format(site, int(ccyy%100), mm, dd)) - dat = np.array(f.variables['Absolute_Humidity']) - dat = dat * 0.001 ##convert to kg/m3 - dat_mask = np.array(f.variables['Absolute_Humidity_mask']) +for site in range(1, NUM_SITES + 1): + f = Dataset('wv_mpd0{}.{:02d}{:02d}{:02d}.dafinal.Python.nc'.format(site, int(CCYY % 100), MM, DD)) + dat = np.array(f.variables['Absolute_Humidity']) + dat = dat * 0.001 ##convert to kg/m3 + dat_mask = np.array(f.variables['Absolute_Humidity_mask']) - nt, nz = dat.shape - tt = np.array(f.variables['time_Absolute_Humidity']) - zz = np.array(f.variables['range_Absolute_Humidity']) + nt, nz = dat.shape + tt = np.array(f.variables['time_Absolute_Humidity']) + zz = np.array(f.variables['range_Absolute_Humidity']) - lat = np.array(f.variables['lidar_latitude']) - lon = np.array(f.variables['lidar_longitude']) - if(lon<0): - lon = 360.0 + lon - elev = np.array(f.variables['lidar_elevation']) + lat = np.array(f.variables['lidar_latitude']) + lon = np.array(f.variables['lidar_longitude']) + if(lon < 0): + lon = 360.0 + lon + elev = np.array(f.variables['lidar_elevation']) - for t in range(nt-1): - ii_sum = int(tt[t]/60) - hh = int(ii_sum/60) - ii = int(ii_sum%60) - for z in range(nz): - if(dat_mask[t,z]==0 and dat[t,z]>=0): ##QC mask and remove negative data - textfile = "work/{:04d}{:02d}{:02d}{:02d}{:02d}".format(ccyy, mm, dd, hh, ii) - f1 = open(textfile, "a") - string = "{:9.5f} {:9.5f} {:8.1f} {:4d} {:02d} {:02d} {:02d} {:02d} {:02d} {:e} {:e}\n".format(lat, lon, elev+zz[z], ccyy, mm, dd, hh, ii, 0, dat[t, z], obs_err) - f1.write(string) - f1.close() + for t in range(nt-1): + ii_sum = int(tt[t] / 60) + hh = int(ii_sum / 60) + ii = int(ii_sum % 60) + for z in range(nz): + if(dat_mask[t, z] == 0 and dat[t, z] >= 0): ##QC mask and remove negative data + textfile = "work/{:04d}{:02d}{:02d}{:02d}{:02d}".format(CCYY, MM, DD, hh, ii) + f1 = open(textfile, "a") + string = "{:9.5f} {:9.5f} {:8.1f} {:4d} {:02d} {:02d} {:02d} {:02d} {:02d} {:e} {:e}\n".format(lat, lon, elev + zz[z], CCYY, MM, DD, hh, ii, 0, dat[t, z], OBS_ERR) + f1.write(string) + f1.close() From 149a0443d9f54d441a7db55661fd1db50855a95e Mon Sep 17 00:00:00 2001 From: Tim Hoar Date: Tue, 28 Apr 2020 14:05:44 -0600 Subject: [PATCH 16/16] Decided to put data on the web. --- observations/obs_converters/MPD/README.md | 27 +++++------------------ 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/observations/obs_converters/MPD/README.md b/observations/obs_converters/MPD/README.md index 96d49ec7b6..2fe2d89142 100644 --- a/observations/obs_converters/MPD/README.md +++ b/observations/obs_converters/MPD/README.md @@ -12,32 +12,15 @@ signals between the online and offline wavelengths is proportional to the amount of water vapor in the atmosphere, which allows the retrieval of absolute humidity profiles above the lidar site. -**THIS** - This observation converter takes absolute humidity (g/m3) profiles -retrieved from the MPD data and convert to format used by DART. -A sample netCDF data for the retrieved MPD absolute humidity is provided -in `data/wv_mpd01.190614.dafinal.Python.nc`. - +retrieved from the MPD data and converts them to the format used by DART. The `obs_converter/MPD/work/convert_to_text.py` script reads the netCDF -files from each MPD site and combine them into text files, one for each -date/time. An example text file is provided in at `data/text.txt`. -The `obs_converter/MPD/work/mpd_text_to_obs` program translates the text -files to DART `obs_seq.out` format. - -**OR** - -This observation converter takes absolute humidity (g/m3) profiles -retrieved from the MPD data and convert to format used by DART. -The `obs_converter/MPD/work/convert_to_text.py` script reads the netCDF -files from each MPD site and combine them into text files, one for each -date/time. The `obs_converter/MPD/work/MPD_text_to_obs` program -translates the text files to DART `obs_seq.out` format. +files from each MPD site and combines them into text files, one for each +date and time. The `obs_converter/MPD/work/MPD_text_to_obs` program +translates the text files to the DART `obs_seq.out` format. Test data for a single site and an example output can be downloaded from -https://www.image.ucar.edu/pub/DART/MPD.tar.gz - -**THAT** +https://www.image.ucar.edu/pub/DART/MPD/MPD.tar.gz For more details of the retrieval and quality control process, and inquire about data availability for your research project,