diff --git a/.gitignore b/.gitignore index e4f75e7..147841a 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,8 @@ response-information/attenuation-data/ response-information/detector-response-data/ response-information/effective-area-data/ response-information/quantum-efficiency-data/ -response-information/atmospheric-data/ \ No newline at end of file +response-information/atmospheric-data/ + +# environment +.env/ +.venv/ diff --git a/response-information/effective-area-data/README.md b/response-information/effective-area-data/README.md index 92c38f0..43d64b8 100644 --- a/response-information/effective-area-data/README.md +++ b/response-information/effective-area-data/README.md @@ -1,3 +1,7 @@ # Effective area data files Only really here so this folder is tracked by `git`. + +## Content +- `nagoya_hxt_onaxis_measurement_v1.txt` is the effective area data for the position 4 optic. +- `nagoya_sxt_onaxis_measurement_v1.txt` is the data for the position 1 optic, and includes OBFs and collimators. diff --git a/response-information/info.yaml b/response-information/info.yaml index 809d839..46b0e59 100644 --- a/response-information/info.yaml +++ b/response-information/info.yaml @@ -1,5 +1,35 @@ ---- - files: - marshall 10-shell: "" - marshall 2-shell: "" - nagoya 1-shell: "" \ No newline at end of file +files: + optics: + eff_area_telescope-2-pan_msfc_heritage: "effective-area-data/FOXSI3_Module_X-7_EA_pan_v1.txt" + eff_area_telescope-2-tilt_msfc_heritage: "effective-area-data/FOXSI3_Module_X-7_EA_tilt_v1.txt" + eff_area_telescope-5-pan_msfc_heritage: "effective-area-data/FOXSI3_Module_X-8_EA_pan_v1.txt" + eff_area_telescope-5-tilt_msfc_heritage: "effective-area-data/FOXSI3_Module_X-8_EA_tilt_v1.txt" + eff_area_msfc_hi_res: "effective-area-data/FOXSI4_Module_MSFC_HiRes_EA_with_models_v1.txt" + eff_area_nagoya_hxt: "effective-area-data/nagoya_hxt_onaxis_measurement_v1.txt" + eff_area_nagoya_sxt: "effective-area-data/nagoya_sxt_onaxis_measurement_v1.txt" + + detectors: + cmos_det_telescope-0_resp: "detector-response-data/cmos/foxsi4_telescope-0_BASIC_RESPONSE_MATRIX_v1.fits" + cmos_det_telescope-1_resp: "detector-response-data/cmos/foxsi4_telescope-1_BASIC_RESPONSE_MATRIX_v1.fits" + + attenuation: + att_thermal_blanket: "attenuation-data/F4_Blanket_transmission_v1.dat" + att_pixelated: "attenuation-data/20240607_fosxi4_transmission_v1.csv" + att_al_mylar: "attenuation-data/thin_mylar_p3_p5_theoretical_v1.csv" + att_telescope-2_uniform_al_cdte: "attenuation-data/unif_att_p2_theoretical_v1.csv" + att_telescope-4_uniform_al_cdte: "attenuation-data/unif_att_p4_theoretical_v1.csv" + att_telescope-0_collimator_ratio: "attenuation-data/foxsi4_telescope-0_BASIC_collimator_aperture_ratio_v1.fits" + att_telescope-1_collimator_ratio: "attenuation-data/foxsi4_telescope-1_BASIC_collimator_aperture_ratio_v1.fits" + att_telescope-0_cmos_obfilter: "attenuation-data/foxsi4_telescope-0_BASIC_optical_blocking_filter_transmittance_v1.fits" + att_telescope-1_cmos_obfilter: "attenuation-data/foxsi4_telescope-1_BASIC_optical_blocking_filter_transmittance_v1.fits" + att_telescope-0_cmos_prefilter: "attenuation-data/foxsi4_telescope-0_BASIC_attenuation_filter_transmittance_v1.fits" + att_telescope-1_cmos_prefilter: "attenuation-data/foxsi4_telescope-1_BASIC_attenuation_filter_transmittance_v1.fits" + att_foxsi4_atmosphere: "attenuation-data/FOXSI4_atmospheric_transmission_v1.fits" + + quantum_efficiency: + qe_cmos_telescope-0: "quantum-efficiency-data/foxsi4_telescope-0_BASIC_sensor_quantum_efficiency_v1.fits" + qe_cmos_telescope-1: "quantum-efficiency-data/foxsi4_telescope-1_BASIC_sensor_quantum_efficiency_v1.fits" + +# remote URL for fetching data. +# can validly append any filename above to the server name to access. +remote_server: "http://foxsi.space.umn.edu/data/response/response-components" diff --git a/response_tools/io/fetch_response_data.py b/response_tools/io/fetch_response_data.py new file mode 100644 index 0000000..7c67718 --- /dev/null +++ b/response_tools/io/fetch_response_data.py @@ -0,0 +1,224 @@ +""" +Script to download data from FOXSI server. +""" + +import os, cmd, sys +# from pathlib import PureWindowsPath, PurePosixPath +import urllib.request +from urllib.parse import urljoin +from tqdm import tqdm +from enum import Enum +from response_tools.io.load_yaml import load_response_context +import inquirer + +import pprint + +local_prefix = 'response-information' + +ignore_urls = ['@eaDir/'] + +DEBUG = True + +def print_green(txt:str): + print('\033[92m' + txt + '\033[0m') +def print_red(txt:str): + print('\033[91m' + txt + '\033[0m') + +class DownloadType(Enum): + latest = 1 + historical = 2 + component = 3 + telescope = 4 + file = 5 + + + +class DownloadPrompt: + def __init__(self): + self.remote = "http://foxsi.space.umn.edu/data/response/response-components/" + self.local_prefix = 'response-information' + self.start_prompt = [ + inquirer.List( + "query", + message="Which data products would you like to download?", + choices=[ + ("Get all latest products", DownloadType.latest), + ("Get all historical products", DownloadType.historical), + ("Get specific response components", DownloadType.component), + ("Get response components for telescope", DownloadType.telescope), + ("Get specific file", DownloadType.file) + ] + ) + ] + self.theme = inquirer.themes.load_theme_from_dict({ + "Question": { + "mark_color": "blue", + "brackets_color": "normal" + }, + "List":{ + "selection_color": "bold_green", + "selection_cursor": ">" + } + }) + self.prompt_machine = { + DownloadType.latest: self._fetch_latest, + DownloadType.historical: self._fetch_historical, + DownloadType.component: self._prompt_component, + DownloadType.telescope: self._prompt_telecope, + DownloadType.file: self._prompt_file + } + + answers = inquirer.prompt(self.start_prompt, theme=self.theme) + + self._handle_prompt(answers["query"]) + + def _handle_prompt(self, reply:DownloadType): + if reply in self.prompt_machine.keys(): + self.prompt_machine[reply]() + else: + raise KeyError("Unimplemented user selection: " + reply) + + def _fetch_latest(self): + print("latest") + def _fetch_historical(self): + print("historical") + def _prompt_component(self): + component_prompt = [ + inquirer.Checkbox( + "component", + message="Which response component(s) would you like to download?", + choices=["detector response", "attenuation", "effective area", "quantum efficiency"] + ) + ] + answers = inquirer.prompt(component_prompt, theme=self.theme) + + + def _prompt_telecope(self): + telescope_prompt = [ + inquirer.Checkbox( + "telescope", + message="Which telescope's products would you like to download?", + choices=[ + ("P0 - CMOS detector, 2-shell optic", 0), + ("P1 - CMOS detector, 1-shell optic", 1), + ("P2 - CdTe detector, 10-shell optic", 2), + ("P3 - CdTe detector, 2-shell optic", 3), + ("P4 - CdTe detector, 1-shell optic", 4), + ("P5 - CdTe detector, 10-shell optic", 5), + ("P6 - Timepix detector, 2-shell optic", 6) + ] + ) + ] + answers = inquirer.prompt(telescope_prompt, theme=self.theme) + + def _prompt_file(self): + file_prompt = [ + inquirer.Text( + "file", + message="Provide the URL to download" + ) + ] + answers = inquirer.prompt(file_prompt, theme=self.theme) + print(answers["file"]) + + + +def green_str(text:str): + return "\033[92m" + text + "\033[0m" + +def foxsi4_download_required(replace_existing=False, verbose=False): + """Download all response component files specified in `response-information/info.yaml`. + + Download data products from a remote server to the local filesystem. Retrieves server + URL and all local paths for saving data from a config file: + `response-tools/response-information/info.yaml`. All downloaded response data will be + saved under `response-tools/response-information`. + + Parameters + ---------- + replace_existing : `bool` + Whether to replace local files with newer versions, if newer versions are + downloaded. Currently throws `NotImplementedError`. + + verbose : `bool` + Toggle for printing verbosely. If `True`, download progress indicators and + filenames are displayed. If `False`, nothing is printed at all. + + Returns + ------- + : `downloaded` + A dict of downloaded data. Keys are the same file identifiers from the YAML + source. Values are the absolute paths on the local filesystem to the downloaded + file. Files which were already existed in the local filesystem (required no + downloaded) are not included in the return value. + """ + + if replace_existing == True: + raise NotImplementedError("No support yet for replacement of old file versions.") + + # print if the verbose flag is set: + def verbose_print(*something): + if verbose: + print(*something) + + req = load_response_context() + server_url = req["remote_server"] + + # for urllib.parse.urljoin to work correctly, server path prefix must end in `/`: + if server_url[-1] != "/": + server_url += "/" + + # directory on local filesystem for saving data: + local_info_dir = os.path.abspath(os.path.join(__file__, "..", "..", "..", "response-information")) + verbose_print("Retrieving response products from:", green_str(server_url)) + verbose_print("Saving response products to:", green_str(local_info_dir)) + + # record which files already exist on-disk (don't waste time downloading): + existing_files = [] + for r,_,fs in os.walk(local_info_dir): + for f in fs: + existing_files.append(os.path.join(r,f)) + + desired_files = [] # list of the files to download + destination_path = [] # local path to save them to + source_name = [] # identifier of the file (YAML key) + do_get = [] # flag whether to download (if the file already exists locally) + + for comp_name in req["files"].keys(): + for f_name, suffix in req["files"][comp_name].items(): + desired_files.append(urljoin(server_url, suffix)) + dest = os.path.join(local_info_dir, suffix) + destination_path.append(dest) + source_name.append(f_name) + + if os.path.exists(dest): + do_get.append(False) + else: + do_get.append(True) + + downloaded = {} + if any(do_get): + verbose_print("Retrieving files...") + for (k, f) in enumerate(tqdm(desired_files, disable=not verbose)): + if do_get[k]: + try: + # create the folders along the save path, if needed + os.makedirs(os.path.dirname(destination_path[k])) + except: + pass + + # download the file: + fname, head = urllib.request.urlretrieve(f, destination_path[k]) + # record the identifier and path of the downloaded file: + downloaded[source_name[k]] = fname + if verbose: + tqdm.write("Downloaded " + green_str(os.path.basename(fname))) + else: + verbose_print("Found nothing new to download") + return downloaded + +if __name__ == "__main__": + # DownloadPrompt() + + downloaded = foxsi4_download_required(verbose=True) + pprint.pprint(downloaded) \ No newline at end of file diff --git a/setup.py b/setup.py index 7c8254c..3450c0d 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,8 @@ "astropy", "pytest", "pandas", + "beautifulsoup4", + "inquirer" ], packages=setuptools.find_packages(), zip_safe=False,