WIP: Incorporate Downloader and CMOS Products

.gitignore

@@ -10,11 +10,11 @@ response_tools.egg-info/
 build/
 
 # data directories
-response-information/attenuation-data/
-response-information/detector-response-data/
-response-information/effective-area-data/
-response-information/quantum-efficiency-data/
-response-information/atmospheric-data/
+response_tools/response-information/attenuation-data/
+response_tools/response-information/detector-response-data/
+response_tools/response-information/effective-area-data/
+response_tools/response-information/quantum-efficiency-data/
+response_tools/response-information/atmospheric-data/
 
 # environment
 .env/

response_tools/__init__.py

@@ -10,6 +10,9 @@
     The information stored in a YAML file that includes information from
     the flight.
 
+`~response_tools.responseFilePath`
+    The response data file location.
+
 `~response_tools.__version__`
     The current version of the code as stated in the setup script.
 
@@ -27,8 +30,12 @@
 ...
 """
 
+import os
+import pathlib
+
 from .version import __version__
 from response_tools.io.load_yaml import load_response_context
 
 # for global context info
 contextResponseInfo = load_response_context()
+responseFilePath = os.path.join(pathlib.Path(__file__).parent, "response-information")

response_tools/attenuation.py

@@ -13,10 +13,11 @@
 import pandas
 import scipy
 
+import response_tools
 from response_tools.util import BaseOutput, native_resolution
 
-ATT_PATH = os.path.join(pathlib.Path(__file__).parent, "..", "response-information", "attenuation-data")
-ATM_PATH = os.path.join(pathlib.Path(__file__).parent, "..", "response-information", "atmospheric-data")
+FILE_PATH = response_tools.responseFilePath
+RESPONSE_INFO_TYPE = response_tools.contextResponseInfo["files"]["attenuation"]
 ASSETS_PATH = os.path.join(pathlib.Path(__file__).parent, "..", "assets", "response-tools-figs", "att-figs")
 
 @dataclass
@@ -57,7 +58,7 @@ def att_thermal_blanket(mid_energies, file=None):
         transmissions, and more. See accessible information using 
         `.contents` on the output.
     """
-    _f = os.path.join(ATT_PATH, "F4_Blanket_transmission_v1.dat") if file is None else file
+    _f = os.path.join(FILE_PATH, RESPONSE_INFO_TYPE["att_thermal_blanket"]) if file is None else file
     att = scipy.io.readsav(_f)
     att_es, att_values = att["energy_kev"] << u.keV, att["f4_transmission"] << u.dimensionless_unscaled
     mid_energies = native_resolution(native_x=att_es, input_x=mid_energies)
@@ -105,7 +106,7 @@ def att_uniform_al_cdte(mid_energies, position=None, file=None):
     """
     if (position is None) or (position not in [2,4]):
         logging.warning(f"The {sys._getframe().f_code.co_name} `position` must be 2 or 4.")
-    _f = os.path.join(ATT_PATH, f"unif_att_p{position}_theoretical_v1.csv") if file is None else file
+    _f = os.path.join(FILE_PATH, RESPONSE_INFO_TYPE[f"att_telescope-{position}_uniform_al_cdte"]) if file is None else file
     att = pandas.read_csv(_f)
     att_es, att_values = att["energy[keV]"] << u.keV, att["transmission"] << u.dimensionless_unscaled
     mid_energies = native_resolution(native_x=att_es, input_x=mid_energies)
@@ -151,7 +152,7 @@ def att_pixelated(mid_energies, use_model=False, file=None):
         transmissions, and more. See accessible information using 
         `.contents` on the output.
     """
-    _f = os.path.join(ATT_PATH, "20240607_fosxi4_transmission_v1.csv") if file is None else file
+    _f = os.path.join(FILE_PATH, RESPONSE_INFO_TYPE["att_pixelated"]) if file is None else file
     att = pandas.read_csv(_f)
     att_es, att_values_measured, att_values_modelled = att["energy"] << u.keV, att["measured_transmission"] << u.dimensionless_unscaled, att["modeled_transmission"] << u.dimensionless_unscaled
 
@@ -197,7 +198,7 @@ def att_al_mylar(mid_energies, file=None):
         transmissions, and more. See accessible information using 
         `.contents` on the output.
     """
-    _f = os.path.join(ATT_PATH, "thin_mylar_p3_p5_theoretical_v1.csv") if file is None else file
+    _f = os.path.join(FILE_PATH, RESPONSE_INFO_TYPE["att_al_mylar"]) if file is None else file
     att = pandas.read_csv(_f)
     att_es, att_values = att["energy[keV]"] << u.keV, att["transmission"] << u.dimensionless_unscaled
     mid_energies = native_resolution(native_x=att_es, input_x=mid_energies)
@@ -224,7 +225,7 @@ def _att_old_prefilter(mid_energies, position=None, file=None):
     if (position is None) or (position not in [0,1]):
         logging.warning(f"The {sys._getframe().f_code.co_name} `position` must be 0 or 1.")
     logging.warning(f"Caution: This might not be the function you are looking for ({sys._getframe().f_code.co_name}), please see `att_cmos_obfilter`.")
-    _f = os.path.join(ATT_PATH, "CMOST_Prefilter_transmission.dat") if file is None else file
+    _f = os.path.join(FILE_PATH, RESPONSE_INFO_TYPE["att_early_cmos_prefilter"]) if file is None else file
     att = scipy.io.readsav(_f)
     att_es, att_values = att["cmos_prefilter_transmission"]["energy_kev"][0] << u.keV, att["cmos_prefilter_transmission"][f"position{position}"][0] << u.dimensionless_unscaled
 
@@ -304,7 +305,7 @@ def att_cmos_filter(mid_energies, telescope=None, file=None):
     if (telescope is None) or (telescope not in [0,1]):
         logging.warning(f"The `telescope` input in {sys._getframe().f_code.co_name} must be 0 or 1.")
         return
-    _f = os.path.join(ATT_PATH, f"foxsi4_telescope-{telescope}_BASIC_attenuation_filter_transmittance_v1.fits") if file is None else file
+    _f = os.path.join(FILE_PATH, RESPONSE_INFO_TYPE[f"att_telescope-{telescope}_cmos_prefilter"]) if file is None else file
     with fits.open(_f) as hdul:
         es, t = hdul[2].data << u.keV, hdul[1].data << u.dimensionless_unscaled
     mid_energies = native_resolution(native_x=es, input_x=mid_energies)
@@ -352,7 +353,7 @@ def att_cmos_obfilter(mid_energies, telescope=None, file=None):
     if (telescope is None) or (telescope not in [0,1]):
         logging.warning(f"The `telescope` input in {sys._getframe().f_code.co_name} must be 0 or 1.")
         return
-    _f = os.path.join(ATT_PATH, f"foxsi4_telescope-{telescope}_BASIC_optical_blocking_filter_transmittance_v1.fits") if file is None else file
+    _f = os.path.join(FILE_PATH, RESPONSE_INFO_TYPE[f"att_telescope-{telescope}_cmos_obfilter"]) if file is None else file
     with fits.open(_f) as hdul:
         es, t = hdul[2].data << u.keV, hdul[1].data << u.dimensionless_unscaled
     mid_energies = native_resolution(native_x=es, input_x=mid_energies)
@@ -399,7 +400,7 @@ def att_cmos_collimator_ratio(off_axis_angle, telescope=None, file=None):
     if (telescope is None) or (telescope not in [0,1]):
         logging.warning(f"The `telescope` input in {sys._getframe().f_code.co_name} must be 0 or 1.")
         return
-    _f = os.path.join(ATT_PATH, f"foxsi4_telescope-{telescope}_BASIC_collimator_aperture_ratio_v1.fits") if file is None else file
+    _f = os.path.join(FILE_PATH, RESPONSE_INFO_TYPE[f"att_telescope-{telescope}_collimator_ratio"]) if file is None else file
     with fits.open(_f) as hdul:
         oa_angles, aperture_ratio = hdul[2].data << u.arcmin, hdul[1].data << u.dimensionless_unscaled
     off_axis_angle = native_resolution(native_x=oa_angles, input_x=off_axis_angle)
@@ -472,7 +473,7 @@ def att_foxsi4_atmosphere(mid_energies, time_range=None, file=None):
         warnings.warn(f"{sys._getframe().f_code.co_name} `time_range` (convertable to astropy.units.seconds) should be of length 2.")
         return
 
-    _f = os.path.join(ATM_PATH, f"FOXSI4_atmospheric_transmission_v1.fits") if file is None else file
+    _f = os.path.join(FILE_PATH, RESPONSE_INFO_TYPE["att_foxsi4_atmosphere"]) if file is None else file
     with fits.open(_f) as hdul:
         native_times, native_energies, transmission = hdul[1].data["TIME"][0]<<u.second, (hdul[1].data["ENERGY"][0]<<u.eV)<<u.keV, hdul[1].data["ATMOSPHERIC_TRANS"][0]<<u.dimensionless_unscaled
 

response_tools/detector_response.py

@@ -11,9 +11,12 @@
 import astropy.units as u
 import numpy as np
 
+import response_tools
 from response_tools.util import BaseOutput
 
-DET_RESP_PATH = os.path.join(pathlib.Path(__file__).parent, "..", "response-information", "detector-response-data")
+FILE_PATH = response_tools.responseFilePath
+RESPONSE_INFO_TYPE = response_tools.contextResponseInfo["files"]["detectors"]
+DET_RESP_PATH = os.path.join(pathlib.Path(__file__).parent, "response-information", "detector-response-data")
 ASSETS_PATH = os.path.join(pathlib.Path(__file__).parent, "..", "assets", "response-tools-figs", "det-resp-figs")
 
 @dataclass
@@ -176,10 +179,8 @@ def cmos_det_resp(file=None, telescope=None):
     if (telescope is None) or (telescope not in [0,1]):
         logging.warning(f"The `telescope` input in {sys._getframe().f_code.co_name} must be 0 or 1.")
         return
-
-    _f = os.path.join(DET_RESP_PATH, 
-                      "cmos", 
-                      f"foxsi4_telescope-{telescope}_BASIC_RESPONSE_MATRIX_v1.fits") if file is None else file
+
+    _f = os.path.join(FILE_PATH, RESPONSE_INFO_TYPE[f"cmos_det_telescope-{telescope}_resp"]) if file is None else file
 
     with fits.open(_f) as hdul:
         matrix, counts, energy = hdul[1].data<<(u.DN/u.ph), hdul[2].data<<u.DN, hdul[3].data<<u.keV # units?

response_tools/effective_area.py

@@ -14,9 +14,11 @@
 from scipy.interpolate import CloughTocher2DInterpolator
 import pandas
 
+import response_tools
 from response_tools.util import BaseOutput, native_resolution
 
-EFF_PATH = os.path.join(pathlib.Path(__file__).parent, "..", "response-information", "effective-area-data")
+FILE_PATH = response_tools.responseFilePath
+RESPONSE_INFO_TYPE = response_tools.contextResponseInfo["files"]["optics"]
 ASSETS_PATH = os.path.join(pathlib.Path(__file__).parent, "..", "assets", "response-tools-figs", "eff-area-figs")
 
 @dataclass
@@ -73,7 +75,7 @@ def eff_area_msfc_10shell(mid_energies, off_axis_angle=0<<u.arcmin, optic_id=Non
     """
     _id = optic_id if optic_id in ["X-7", "X-8"] else None
     if _id is None:
-        logging.warning("Please provide a MSFC heritage optic ID from [\'X-7\', \'X-8\'].")
+        logging.warning("Please provide a MSFC heritage optic ID from [\'X-7\', \'X-8\'] (telescope 2 and 5, respectively).")
         return
 
     _ft, vals_tilt = _get_ea_file_info(_id, "tilt", file=file_tilt)
@@ -107,7 +109,8 @@ def eff_area_msfc_10shell(mid_energies, off_axis_angle=0<<u.arcmin, optic_id=Non
 
 def _get_ea_file_info(optics_id, axis, file=None):
     """Loads in the desired heritage .txt optic file."""
-    _f = os.path.join(EFF_PATH, f"FOXSI3_Module_{optics_id}_EA_{axis}_v1.txt") if file is None else file
+    id2tel = {"X-7":2, "X-8":5}
+    _f = os.path.join(FILE_PATH, RESPONSE_INFO_TYPE[f"eff_area_telescope-{id2tel[optics_id]}-{axis}_msfc_heritage"]) if file is None else file
     return _f, np.loadtxt(_f, delimiter=",")
 
 def _get_oa_and_ea_msfc_10shell(grid):
@@ -167,7 +170,7 @@ def eff_area_msfc_hi_res(mid_energies, off_axis_angle=None, position=None, use_m
     if off_axis_angle is not None:
         logging.warning(f"The `off_axis_angle` input for MSFC high-resolution optics ({sys._getframe().f_code.co_name}) is not yet implemented.")
     # msfc_hi_res effective areas
-    _f = os.path.join(EFF_PATH, "FOXSI4_Module_MSFC_HiRes_EA_with_models_v1.txt") if file is None else file
+    _f = os.path.join(FILE_PATH, RESPONSE_INFO_TYPE["eff_area_msfc_hi_res"]) if file is None else file
     e, f1, f2, f3, f1_m, f2_m, f3_m = np.loadtxt(_f).T
 
     fm1, fm2, fm3 = (f1_m, f2_m, f3_m) if use_model else (f1, f2, f3)
@@ -207,7 +210,7 @@ def _eff_area_msfc(mid_energies, file=None):
     # msfc_hi_res effective areas
     logging.warning(f"Caution: This might not be the function ({sys._getframe().f_code.co_name}) you are looking for, please see `eff_area_msfc_hi_res`.")
     logging.warning("This current function loads in some very early numbers for the new FOXSI-4 MSFC optics.")
-    _f = os.path.join(EFF_PATH, "3Inner_EA_EPDL97_14AA.csv") if file is None else file
+    _f = os.path.join(FILE_PATH, RESPONSE_INFO_TYPE["eff_area_early_msfc_hi_res"]) if file is None else file
     msfc_hi_res = pandas.read_csv(_f).to_numpy()[:,1:] # remove the first column that only indexes
     # in cm2 ; we use the innermost and the 3rd innermost shells (S10 and S08) [from Yixian]
     msfc_hi_res_es, msfc_hi_res_effas08, msfc_hi_res_effas10 = msfc_hi_res[:,0] << u.keV, msfc_hi_res[:,1] << u.cm**2, msfc_hi_res[:,3] << u.cm**2 
@@ -234,7 +237,7 @@ def _eff_area_nagoya(mid_energies, file=None):
     # nagoya sxr effective areas
     logging.warning(f"Caution: This might not be the function ({sys._getframe().f_code.co_name}) you are looking for and has other effects included than just optics.")
     logging.warning("This current function loads in some very early numbers for the new FOXSI-4 Nagoya SXR optics.")
-    _f = os.path.join(EFF_PATH, "effective-area_raytracing_soft-xray-optic_on-axis.txt") if file is None else file
+    _f = os.path.join(FILE_PATH, RESPONSE_INFO_TYPE["eff_area_early_nagoya_sxt"]) if file is None else file
     nagoya_sxr = np.loadtxt(_f)
     nagoya_sxr_es, nagoya_sxr_effa = nagoya_sxr[:,0] << u.keV, nagoya_sxr[:,1]/100 << u.cm**2
 
@@ -290,12 +293,12 @@ def eff_area_nagoya_hxt(mid_energies, off_axis_angle=None, use_model=False, file
         logging.warning(f"The `off_axis_angle` input for Nagoya high-resolution optics ({sys._getframe().f_code.co_name}) is not yet implemented.")
     # nagoya hxr effective areas
     if not use_model:
-        _f = os.path.join(EFF_PATH, "nagoya_hxt_onaxis_measurement_v1.txt") if file is None else file
+        _f = os.path.join(FILE_PATH, RESPONSE_INFO_TYPE["eff_area_measured_nagoya_hxt"]) if file is None else file
         nagoya_hxr = np.loadtxt(_f)
         nagoya_hxr_es, _, nagoya_hxr_effa, _ = nagoya_hxr[:,0] << u.keV, nagoya_hxr[:,1] << u.keV, nagoya_hxr[:,2] << u.mm**2, nagoya_hxr[:,3] << u.mm**2
         nagoya_hxr_effa = nagoya_hxr_effa << u.cm**2
     else:
-        _f = os.path.join(EFF_PATH, "HXR_Nagoya_FOXSI4.arf") if file is None else file
+        _f = os.path.join(FILE_PATH, RESPONSE_INFO_TYPE["eff_area_modeled_nagoya_hxt"]) if file is None else file
         with fits.open(_f) as hdul:
             nagoya_hxr_es = (hdul[1].data["ENERG_LO"]+hdul[1].data["ENERG_HI"])/2 << u.keV
             nagoya_hxr_effa = hdul[1].data["SPECRESP"] << u.cm**2
@@ -347,12 +350,12 @@ def eff_area_nagoya_sxt(mid_energies, use_model=False, file=None):
     """
     # nagoya sxr effective areas
     if not use_model:
-        _f = os.path.join(EFF_PATH, "nagoya_sxt_onaxis_measurement_v1.txt") if file is None else file
+        _f = os.path.join(FILE_PATH, RESPONSE_INFO_TYPE["eff_area_measured_nagoya_sxt"]) if file is None else file
         nagoya_sxr = np.loadtxt(_f)
         nagoya_sxr_es, _, nagoya_sxr_effa, _ = nagoya_sxr[:,0] << u.keV, nagoya_sxr[:,1] << u.keV, nagoya_sxr[:,2] << u.mm**2, nagoya_sxr[:,3] << u.mm**2
         nagoya_sxr_effa = nagoya_sxr_effa << u.cm**2
     else:
-        _f = os.path.join(EFF_PATH, "SXR_nocollimator_noobf.arf") if file is None else file
+        _f = os.path.join(FILE_PATH, RESPONSE_INFO_TYPE["eff_area_modeled_nagoya_sxt"]) if file is None else file
         with fits.open(_f) as hdul:
             nagoya_sxr_es = (hdul[1].data["ENERG_LO"]+hdul[1].data["ENERG_HI"])/2 << u.keV
             nagoya_sxr_effa = hdul[1].data["SPECRESP"] << u.cm**2
@@ -409,8 +412,7 @@ def eff_area_cmos(mid_energies, telescope=None, file=None):
     if (telescope is None) or (telescope not in [0,1]):
         logging.warning(f"The `telescope` input in {sys._getframe().f_code.co_name} must be 0 or 1.")
         return
-
-    _f = os.path.join(EFF_PATH, f"foxsi4_telescope-{telescope}_BASIC_mirror_effective_area_v1.fits") if file is None else file
+    _f = os.path.join(FILE_PATH, RESPONSE_INFO_TYPE[f"eff_area_cmos_mirror{telescope}"]) if file is None else file
     with fits.open(_f) as hdul:
         es, effas = hdul[2].data << u.keV, hdul[1].data << u.cm**2
     mid_energies = native_resolution(native_x=es, input_x=mid_energies)
@@ -477,7 +479,7 @@ def eff_area_cmos_telescope(mid_energies, telescope=None, file=None):
     logging.warning("If you care about what elements are included then proceed carefully.")
     logging.warning("For current file, see PR#11 in the `cmos-tools` repository.")
 
-    _f = os.path.join(EFF_PATH, f"foxsi4_telescope-{telescope}_BASIC_TELESCOPE_RESPONSE_V25APR13.fits") if file is None else file
+    _f = os.path.join(FILE_PATH, RESPONSE_INFO_TYPE[f"eff_area_cmos_telescope{telescope}"]) if file is None else file
     with fits.open(_f) as hdul:
         # _ is the off-axis angle but it's just [0] at the minute
         ea_energies, _, effas = hdul[2].data << u.keV, hdul[3].data << u.arcsec, hdul[1].data << u.cm**2
@@ -688,7 +690,7 @@ def asset_all_optics(save_asset=False):
         gsax.set_ylim([0, np.nanmax(x8.effective_areas).value*1.01])
 
     gs_ax5 = fig.add_subplot(gs[2, 0])
-    _f = os.path.join(EFF_PATH, "FOXSI4_Module_MSFC_HiRes_EA_with_models_v1.txt")
+    _f = os.path.join(FILE_PATH, RESPONSE_INFO_TYPE["eff_area_msfc_hi_res"])
     e, *_ = np.loadtxt(_f).T
     e <<= u.keV
     msfc_hi_res_p0 = eff_area_msfc_hi_res(e, position=0)

response_tools/io/fetch_response_data.py

@@ -169,7 +169,7 @@ def verbose_print(*something):
         server_url += "/"
 
     # directory on local filesystem for saving data:
-    local_info_dir = os.path.abspath(os.path.join(__file__, "..", "..", "..", "response-information"))
+    local_info_dir = os.path.abspath(os.path.join(__file__, "..", "..", "response-information"))
     verbose_print("Retrieving response products from:", green_str(server_url))
     verbose_print("Saving response products to:", green_str(local_info_dir))
 

response_tools/io/load_yaml.py

@@ -16,4 +16,4 @@ def load_yaml(filename):
 
 def load_response_context():
     """Function to load in the context information from file. """
-    return load_yaml(os.path.join(pathlib.Path(__file__).parent, "..", "..", "response-information", "info.yaml"))
+    return load_yaml(os.path.join(pathlib.Path(__file__).parent, "..", "response-information", "info.yaml"))

response_tools/quantum_efficiency.py

@@ -10,9 +10,11 @@
 import astropy.units as u
 import numpy as np
 
+import response_tools
 from response_tools.util import BaseOutput, native_resolution
 
-Q_PATH = os.path.join(pathlib.Path(__file__).parent, "..", "response-information", "quantum-efficiency-data")
+FILE_PATH = response_tools.responseFilePath
+RESPONSE_INFO_TYPE = response_tools.contextResponseInfo["files"]["quantum_efficiency"]
 ASSETS_PATH = os.path.join(pathlib.Path(__file__).parent, "..", "assets", "response-tools-figs", "quantum-eff-figs")
 
 @dataclass
@@ -62,8 +64,8 @@ def qe_cmos(mid_energies, telescope=None, file=None):
     if (telescope is None) or (telescope not in [0,1]):
         logging.warning(f"The `telescope` input in {sys._getframe().f_code.co_name} must be 0 or 1.")
         return
-        
-    _f = os.path.join(Q_PATH, f"foxsi4_telescope-{telescope}_BASIC_sensor_quantum_efficiency_v1.fits") if file is None else file
+
+    _f = os.path.join(FILE_PATH, RESPONSE_INFO_TYPE[f"qe_cmos_telescope-{telescope}"]) if file is None else file
     with fits.open(_f) as hdul:
         es, qe = hdul[2].data << u.keV, hdul[1].data << u.dimensionless_unscaled
     mid_energies = native_resolution(native_x=es, input_x=mid_energies)