Add YAML-dependent downloader

.gitignore

@@ -14,4 +14,8 @@ response-information/attenuation-data/
 response-information/detector-response-data/
 response-information/effective-area-data/
 response-information/quantum-efficiency-data/
-response-information/atmospheric-data/
+response-information/atmospheric-data/
+
+# environment
+.env/
+.venv/

response-information/effective-area-data/README.md

@@ -1,3 +1,7 @@
 # Effective area data files
 
 Only really here so this folder is tracked by `git`.
+
+## Content
+- `nagoya_hxt_onaxis_measurement_v1.txt` is the effective area data for the position 4 optic.
+- `nagoya_sxt_onaxis_measurement_v1.txt` is the data for the position 1 optic, and includes OBFs and collimators.

response-information/info.yaml

@@ -1,5 +1,35 @@
----
-    files:
-        marshall 10-shell: ""
-        marshall 2-shell: "" 
-        nagoya 1-shell: ""
+files:
+  optics:
+    eff_area_telescope-2-pan_msfc_heritage: "effective-area-data/FOXSI3_Module_X-7_EA_pan_v1.txt"
+    eff_area_telescope-2-tilt_msfc_heritage: "effective-area-data/FOXSI3_Module_X-7_EA_tilt_v1.txt"
+    eff_area_telescope-5-pan_msfc_heritage: "effective-area-data/FOXSI3_Module_X-8_EA_pan_v1.txt"
+    eff_area_telescope-5-tilt_msfc_heritage: "effective-area-data/FOXSI3_Module_X-8_EA_tilt_v1.txt"
+    eff_area_msfc_hi_res: "effective-area-data/FOXSI4_Module_MSFC_HiRes_EA_with_models_v1.txt"
+    eff_area_nagoya_hxt: "effective-area-data/nagoya_hxt_onaxis_measurement_v1.txt"
+    eff_area_nagoya_sxt: "effective-area-data/nagoya_sxt_onaxis_measurement_v1.txt"
+
+  detectors:
+    cmos_det_telescope-0_resp: "detector-response-data/cmos/foxsi4_telescope-0_BASIC_RESPONSE_MATRIX_v1.fits"
+    cmos_det_telescope-1_resp: "detector-response-data/cmos/foxsi4_telescope-1_BASIC_RESPONSE_MATRIX_v1.fits"
+
+  attenuation:
+    att_thermal_blanket: "attenuation-data/F4_Blanket_transmission_v1.dat"
+    att_pixelated: "attenuation-data/20240607_fosxi4_transmission_v1.csv"
+    att_al_mylar: "attenuation-data/thin_mylar_p3_p5_theoretical_v1.csv"
+    att_telescope-2_uniform_al_cdte: "attenuation-data/unif_att_p2_theoretical_v1.csv"
+    att_telescope-4_uniform_al_cdte: "attenuation-data/unif_att_p4_theoretical_v1.csv"
+    att_telescope-0_collimator_ratio: "attenuation-data/foxsi4_telescope-0_BASIC_collimator_aperture_ratio_v1.fits"
+    att_telescope-1_collimator_ratio: "attenuation-data/foxsi4_telescope-1_BASIC_collimator_aperture_ratio_v1.fits"
+    att_telescope-0_cmos_obfilter: "attenuation-data/foxsi4_telescope-0_BASIC_optical_blocking_filter_transmittance_v1.fits"
+    att_telescope-1_cmos_obfilter: "attenuation-data/foxsi4_telescope-1_BASIC_optical_blocking_filter_transmittance_v1.fits"
+    att_telescope-0_cmos_prefilter: "attenuation-data/foxsi4_telescope-0_BASIC_attenuation_filter_transmittance_v1.fits"
+    att_telescope-1_cmos_prefilter: "attenuation-data/foxsi4_telescope-1_BASIC_attenuation_filter_transmittance_v1.fits"
+    att_foxsi4_atmosphere: "attenuation-data/FOXSI4_atmospheric_transmission_v1.fits"
+
+  quantum_efficiency:
+    qe_cmos_telescope-0: "quantum-efficiency-data/foxsi4_telescope-0_BASIC_sensor_quantum_efficiency_v1.fits"
+    qe_cmos_telescope-1: "quantum-efficiency-data/foxsi4_telescope-1_BASIC_sensor_quantum_efficiency_v1.fits"
+
+# remote URL for fetching data.
+# can validly append any filename above to the server name to access.
+remote_server: "http://foxsi.space.umn.edu/data/response/response-components"

response_tools/io/fetch_response_data.py

@@ -0,0 +1,224 @@
+"""
+Script to download data from FOXSI server.
+"""
+
+import os, cmd, sys
+# from pathlib import PureWindowsPath, PurePosixPath
+import urllib.request
+from urllib.parse import urljoin
+from tqdm import tqdm
+from enum import Enum
+from response_tools.io.load_yaml import load_response_context
+import inquirer
+
+import pprint
+
+local_prefix = 'response-information'
+
+ignore_urls = ['@eaDir/']
+
+DEBUG = True
+
+def print_green(txt:str):
+    print('\033[92m' + txt + '\033[0m')
+def print_red(txt:str):
+    print('\033[91m' + txt + '\033[0m')
+
+class DownloadType(Enum):
+    latest = 1
+    historical = 2
+    component = 3
+    telescope = 4
+    file = 5
+
+
+
+class DownloadPrompt:
+    def __init__(self):
+        self.remote = "http://foxsi.space.umn.edu/data/response/response-components/"
+        self.local_prefix = 'response-information'
+        self.start_prompt = [
+            inquirer.List(
+                "query",
+                message="Which data products would you like to download?",
+                choices=[
+                    ("Get all latest products",                 DownloadType.latest),
+                    ("Get all historical products",             DownloadType.historical),
+                    ("Get specific response components",        DownloadType.component),
+                    ("Get response components for telescope",   DownloadType.telescope),
+                    ("Get specific file",                       DownloadType.file)
+                ]
+            )
+        ]
+        self.theme = inquirer.themes.load_theme_from_dict({
+            "Question": {
+                "mark_color": "blue",
+                "brackets_color": "normal"
+            },
+            "List":{
+                "selection_color": "bold_green",
+                "selection_cursor": ">"
+            }
+        })
+        self.prompt_machine = {
+            DownloadType.latest:        self._fetch_latest,
+            DownloadType.historical:    self._fetch_historical,
+            DownloadType.component:     self._prompt_component,
+            DownloadType.telescope:     self._prompt_telecope,
+            DownloadType.file:          self._prompt_file
+        }
+
+        answers = inquirer.prompt(self.start_prompt, theme=self.theme)
+
+        self._handle_prompt(answers["query"])
+
+    def _handle_prompt(self, reply:DownloadType):
+        if reply in self.prompt_machine.keys():
+            self.prompt_machine[reply]()
+        else:
+            raise KeyError("Unimplemented user selection: " + reply)
+
+    def _fetch_latest(self):
+        print("latest")
+    def _fetch_historical(self):
+        print("historical")
+    def _prompt_component(self):
+        component_prompt = [
+            inquirer.Checkbox(
+                "component",
+                message="Which response component(s) would you like to download?",
+                choices=["detector response", "attenuation", "effective area", "quantum efficiency"]
+            )
+        ]
+        answers = inquirer.prompt(component_prompt, theme=self.theme)
+
+
+    def _prompt_telecope(self):
+        telescope_prompt = [
+            inquirer.Checkbox(
+                "telescope",
+                message="Which telescope's products would you like to download?",
+                choices=[
+                    ("P0 - CMOS detector,    2-shell optic", 0),
+                    ("P1 - CMOS detector,    1-shell optic", 1),
+                    ("P2 - CdTe detector,   10-shell optic", 2),
+                    ("P3 - CdTe detector,    2-shell optic", 3),
+                    ("P4 - CdTe detector,    1-shell optic", 4),
+                    ("P5 - CdTe detector,   10-shell optic", 5),
+                    ("P6 - Timepix detector, 2-shell optic", 6)
+                ]
+            )
+        ]
+        answers = inquirer.prompt(telescope_prompt, theme=self.theme)
+
+    def _prompt_file(self):
+        file_prompt = [
+            inquirer.Text(
+                "file",
+                message="Provide the URL to download"
+            )
+        ]
+        answers = inquirer.prompt(file_prompt, theme=self.theme)
+        print(answers["file"])
+
+
+
+def green_str(text:str):
+    return "\033[92m" + text + "\033[0m"
+
+def foxsi4_download_required(replace_existing=False, verbose=False):
+    """Download all response component files specified in `response-information/info.yaml`.
+
+        Download data products from a remote server to the local filesystem. Retrieves server
+        URL and all local paths for saving data from a config file:
+        `response-tools/response-information/info.yaml`. All downloaded response data will be
+        saved under `response-tools/response-information`.
+
+        Parameters
+        ----------
+        replace_existing : `bool`
+            Whether to replace local files with newer versions, if newer versions are
+            downloaded. Currently throws `NotImplementedError`.  
+
+        verbose : `bool`
+            Toggle for printing verbosely. If `True`, download progress indicators and
+            filenames are displayed. If `False`, nothing is printed at all.
+
+        Returns
+        -------    
+        : `downloaded`
+            A dict of downloaded data. Keys are the same file identifiers from the YAML
+            source. Values are the absolute paths on the local filesystem to the downloaded
+            file. Files which were already existed in the local filesystem (required no
+            downloaded) are not included in the return value.
+    """
+
+    if replace_existing == True:
+        raise NotImplementedError("No support yet for replacement of old file versions.")
+
+    # print if the verbose flag is set:
+    def verbose_print(*something):
+        if verbose:
+            print(*something)
+
+    req = load_response_context()
+    server_url = req["remote_server"]
+
+    # for urllib.parse.urljoin to work correctly, server path prefix must end in `/`:
+    if server_url[-1] != "/":
+        server_url += "/"
+
+    # directory on local filesystem for saving data:
+    local_info_dir = os.path.abspath(os.path.join(__file__, "..", "..", "..", "response-information"))
+    verbose_print("Retrieving response products from:", green_str(server_url))
+    verbose_print("Saving response products to:", green_str(local_info_dir))
+
+    # record which files already exist on-disk (don't waste time downloading):
+    existing_files = []
+    for r,_,fs in os.walk(local_info_dir):
+        for f in fs:
+            existing_files.append(os.path.join(r,f))
+
+    desired_files = []      # list of the files to download
+    destination_path = []   # local path to save them to
+    source_name = []        # identifier of the file (YAML key)
+    do_get = []             # flag whether to download (if the file already exists locally)
+
+    for comp_name in req["files"].keys():
+        for f_name, suffix in req["files"][comp_name].items():
+            desired_files.append(urljoin(server_url, suffix))
+            dest = os.path.join(local_info_dir, suffix)
+            destination_path.append(dest)
+            source_name.append(f_name)
+
+            if os.path.exists(dest):
+                do_get.append(False)
+            else:
+                do_get.append(True)
+
+    downloaded = {}
+    if any(do_get):
+        verbose_print("Retrieving files...")
+        for (k, f) in enumerate(tqdm(desired_files, disable=not verbose)):
+            if do_get[k]:
+                try:
+                    # create the folders along the save path, if needed
+                    os.makedirs(os.path.dirname(destination_path[k]))
+                except:
+                    pass
+
+                # download the file:
+                fname, head = urllib.request.urlretrieve(f, destination_path[k])
+                # record the identifier and path of the downloaded file:
+                downloaded[source_name[k]] = fname
+                if verbose:
+                    tqdm.write("Downloaded " + green_str(os.path.basename(fname)))
+    else:
+        verbose_print("Found nothing new to download")
+    return downloaded
+
+if __name__ == "__main__":
+    # DownloadPrompt()
+
+    downloaded = foxsi4_download_required(verbose=True)
+    pprint.pprint(downloaded)

setup.py

@@ -14,6 +14,8 @@
             "astropy",
             "pytest",
             "pandas",
+            "beautifulsoup4",
+            "inquirer"
         ],
     packages=setuptools.find_packages(),
     zip_safe=False,