From 31255ee82cbf7b6d902f9723bcc23f0b8823266d Mon Sep 17 00:00:00 2001 From: Thanasi Pantazides Date: Mon, 15 Sep 2025 11:14:51 -0500 Subject: [PATCH 1/6] moving to other computer --- .../response_tools_py/io/fetch_response_data.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 response-tools-py/response_tools_py/io/fetch_response_data.py diff --git a/response-tools-py/response_tools_py/io/fetch_response_data.py b/response-tools-py/response_tools_py/io/fetch_response_data.py new file mode 100644 index 0000000..c1df1b8 --- /dev/null +++ b/response-tools-py/response_tools_py/io/fetch_response_data.py @@ -0,0 +1,8 @@ +""" +Script to download data from FOXSI server. +""" + +import os, cmd, sys + +if __name__ == "__main__": + pass From 52a8016ade5275d8fed682825abac8c37c65ab67 Mon Sep 17 00:00:00 2001 From: thanasipantazides Date: Wed, 24 Sep 2025 14:59:38 -0500 Subject: [PATCH 2/6] thanasi downloader wip --- .gitignore | 6 +- .../effective-area-data/README.md | 4 + response-information/info.yaml | 12 +- .../io/fetch_response_data.py | 175 +++++++++++++++++- 4 files changed, 190 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index e4aac3a..ac8f2f3 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,8 @@ response_tools_py.egg-info/ response-information/attenuation-data/ response-information/detector-response-data/ response-information/effective-area-data/ -response-information/quantum-efficiency-data/ \ No newline at end of file +response-information/quantum-efficiency-data/ + +# environment +.env/ +.venv/ diff --git a/response-information/effective-area-data/README.md b/response-information/effective-area-data/README.md index 92c38f0..43d64b8 100644 --- a/response-information/effective-area-data/README.md +++ b/response-information/effective-area-data/README.md @@ -1,3 +1,7 @@ # Effective area data files Only really here so this folder is tracked by `git`. + +## Content +- `nagoya_hxt_onaxis_measurement_v1.txt` is the effective area data for the position 4 optic. +- `nagoya_sxt_onaxis_measurement_v1.txt` is the data for the position 1 optic, and includes OBFs and collimators. diff --git a/response-information/info.yaml b/response-information/info.yaml index 809d839..5e90416 100644 --- a/response-information/info.yaml +++ b/response-information/info.yaml @@ -1,5 +1,7 @@ ---- - files: - marshall 10-shell: "" - marshall 2-shell: "" - nagoya 1-shell: "" \ No newline at end of file +files: + attenuation: + att_telescope-0_cmos_obfilter: "attenuation-data/foxsi4_telescope-0_BASIC_optical_blocking_filter_transmittance_v1.fits" + +# remote URL for fetching data. +# can validly append any filename above to the server name to access. +remote_server: "http://foxsi.space.umn.edu/data/response/response-components" diff --git a/response-tools-py/response_tools_py/io/fetch_response_data.py b/response-tools-py/response_tools_py/io/fetch_response_data.py index c1df1b8..e76a043 100644 --- a/response-tools-py/response_tools_py/io/fetch_response_data.py +++ b/response-tools-py/response_tools_py/io/fetch_response_data.py @@ -2,7 +2,180 @@ Script to download data from FOXSI server. """ +import urllib.request import os, cmd, sys +from enum import Enum +import inquirer + +from bs4 import BeautifulSoup as bs +from astropy.io import fits + +import pprint + +remote = "http://foxsi.space.umn.edu/data/response/response-components/" +local_prefix = 'response-information' +area_files = [ + "http://foxsi.space.umn.edu/data/response/response-components/effective-area-data/foxsi4_telescope-0_BASIC_mirror_effective_area_v1.fits", + "http://foxsi.space.umn.edu/data/response/response-components/effective-area-data/foxsi4_telescope-1_BASIC_mirror_effective_area_v1.fits", + "http://foxsi.space.umn.edu/data/response/response-components/effective-area-data/nagoya_hxt_onaxis_measurement_v1.txt", + "http://foxsi.space.umn.edu/data/response/response-components/effective-area-data/nagoya_sxt_onaxis_measurement_v1.txt", + "http://foxsi.space.umn.edu/data/response/response-components/effective-area-data/FOXSI4_Module_MSFC_HiRes_EA_with_models_v1.txt", + "http://foxsi.space.umn.edu/data/response/response-components/effective-area-data/foxsi4_telescope-0_BASIC_TELESCOPE_RESPONSE_V25APR13.fits", + "http://foxsi.space.umn.edu/data/response/response-components/effective-area-data/foxsi4_telescope-1_BASIC_TELESCOPE_RESPONSE_V25APR13.fits" +] + +ignore_urls = ['@eaDir/'] + +DEBUG = True + +def print_green(txt:str): + print('\033[92m' + txt + '\033[0m') +def print_red(txt:str): + print('\033[91m' + txt + '\033[0m') + +class DownloadType(Enum): + latest = 1 + historical = 2 + component = 3 + telescope = 4 + file = 5 + + + +class DownloadPrompt: + def __init__(self): + self.remote = "http://foxsi.space.umn.edu/data/response/response-components/" + self.local_prefix = 'response-information' + self.start_prompt = [ + inquirer.List( + "query", + message="Which data products would you like to download?", + choices=[ + ("Get all latest products", DownloadType.latest), + ("Get all historical products", DownloadType.historical), + ("Get specific response components", DownloadType.component), + ("Get response components for telescope", DownloadType.telescope), + ("Get specific file", DownloadType.file) + ] + ) + ] + self.theme = inquirer.themes.load_theme_from_dict({ + "Question": { + "mark_color": "blue", + "brackets_color": "normal" + }, + "List":{ + "selection_color": "bold_green", + "selection_cursor": ">" + } + }) + self.prompt_machine = { + DownloadType.latest: self._fetch_latest, + DownloadType.historical: self._fetch_historical, + DownloadType.component: self._prompt_component, + DownloadType.telescope: self._prompt_telecope, + DownloadType.file: self._prompt_file + } + + answers = inquirer.prompt(self.start_prompt, theme=self.theme) + + self._handle_prompt(answers["query"]) + + def _handle_prompt(self, reply:DownloadType): + if reply in self.prompt_machine.keys(): + self.prompt_machine[reply]() + else: + raise KeyError("Unimplemented user selection: " + reply) + + def _fetch_latest(self): + print("latest") + def _fetch_historical(self): + print("historical") + def _prompt_component(self): + component_prompt = [ + inquirer.Checkbox( + "component", + message="Which response component(s) would you like to download?", + choices=["detector response", "attenuation", "effective area", "quantum efficiency"] + ) + ] + answers = inquirer.prompt(component_prompt, theme=self.theme) + + + def _prompt_telecope(self): + telescope_prompt = [ + inquirer.Checkbox( + "telescope", + message="Which telescope's products would you like to download?", + choices=[ + ("P0 - CMOS detector, 2-shell optic", 0), + ("P1 - CMOS detector, 1-shell optic", 1), + ("P2 - CdTe detector, 10-shell optic", 2), + ("P3 - CdTe detector, 2-shell optic", 3), + ("P4 - CdTe detector, 1-shell optic", 4), + ("P5 - CdTe detector, 10-shell optic", 5), + ("P6 - Timepix detector, 2-shell optic", 6) + ] + ) + ] + answers = inquirer.prompt(telescope_prompt, theme=self.theme) + + def _prompt_file(self): + file_prompt = [ + inquirer.Text( + "file", + message="Provide the URL to download" + ) + ] + answers = inquirer.prompt(file_prompt, theme=self.theme) + print(answers["file"]) + if __name__ == "__main__": - pass + DownloadPrompt() + # pprint.pprint(answers) + + # if answers["query"] == "" + + + # for file in area_files: + # url_suffix = os.path.join(os.path.split(file)[-2], os.path.split(file)[-1]) + # url_dir_prefix, url_file = os.path.split(file) + # _, url_dir = os.path.split(url_dir_prefix) + + # local_dest = os.path.abspath(os.path.join(__file__, '..', '..', '..', '..', local_prefix, url_dir, url_file)) + # if os.path.isfile(local_dest): + # print_red('\talready have local copy of ' + url_file + ', skipping.') + # continue + + # local_name, local_ext = os.path.splitext(local_dest) + # local_ver = local_name.rsplit('v', 1) + + # print('found new file: ' + local_dest) + # print_green('\tfound version: ' + local_ver[-1]) + # print_green('\t' + local_dest) + + # fname, head = urllib.request.urlretrieve(file, local_dest) + # # d = f.read() + + # if os.path.splitext(fname) == '.fits': + # with fits.open(fname) as hdul: + # print(hdul.info()) + # elif os.path.splitext(fname) == '.csv': + # print('a csv...') + + + + # f = urllib.request.urlopen(remote) + # d = f.read() + + # soup = bs(d, 'html.parser') + # print('reading page:', '\033[92m' + soup.title.string + '\033[0m ...') + + # links = soup.find_all('a') # select all HTML tags + # print('found links:') + # for link in links: # check each link + # h = link.get('href') # get the hyperref + # if h[-1] == '/': # find folders (end in `/`) + # if h not in ignore_urls: # ignore the ignorables + # print_green('\t' + h) From cde679dc473f6926e448b5e1455d50edaecb3421 Mon Sep 17 00:00:00 2001 From: thanasipantazides Date: Wed, 24 Sep 2025 16:13:38 -0500 Subject: [PATCH 3/6] add structured YAML with names for each server file --- response-information/info.yaml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/response-information/info.yaml b/response-information/info.yaml index 5e90416..51691b6 100644 --- a/response-information/info.yaml +++ b/response-information/info.yaml @@ -1,6 +1,34 @@ files: + optics: + eff_area_telescope-2-pan_msfc_heritage: "effective-area-data/FOXSI3_Module_X-7_EA_pan_v1.txt" + eff_area_telescope-2-tilt_msfc_heritage: "effective-area-data/FOXSI3_Module_X-7_EA_pan_v1.txt" + eff_area_telescope-5-pan_msfc_heritage: "effective-area-data/FOXSI3_Module_X-8_EA_pan_v1.txt" + eff_area_telescope-5-tilt_msfc_heritage: "effective-area-data/FOXSI3_Module_X-8_EA_pan_v1.txt" + eff_area_msfc_hi_res: "effective-area-data/FOXSI4_Module_MSFC_HiRes_EA_with_models_v1.txt" + eff_area_nagoya_hxt: "effective-area-data/nagoya_hxt_onaxis_measurement_v1.txt" + eff_area_nagoya_sxt: "effective-area-data/nagoya_sxt_onaxis_measurement_v1.txt" + + detectors: + cmos_det_telescope-0_resp: "detector-response-data/cmos/foxsi4_telescope-0_BASIC_RESPONSE_MATRIX_v1.fits" + cmos_det_telescope-1_resp: "detector-response-data/cmos/foxsi4_telescope-1_BASIC_RESPONSE_MATRIX_v1.fits" + attenuation: + att_thermal_blanket: "attenuation-data/F4_Blanket_transmission_v1.dat" + att_pixelated: "attenuation-data/20240607_fosxi4_transmission_v1.csv" + att_al_mylar: "attenuation-data/thin_mylar_p3_p5_theoretical_v1.csv" + att_telescope-2_uniform_al_cdte: "attenuation-data/unif_att_p2_theoretical_v1.csv" + att_telescope-4_uniform_al_cdte: "attenuation-data/unif_att_p4_theoretical_v1.csv" + att_telescope-0_collimator_ratio: "attenuation-data/foxsi4_telescope-0_BASIC_collimator_aperture_ratio_v1.fits" + att_telescope-1_collimator_ratio: "attenuation-data/foxsi4_telescope-1_BASIC_collimator_aperture_ratio_v1.fits" att_telescope-0_cmos_obfilter: "attenuation-data/foxsi4_telescope-0_BASIC_optical_blocking_filter_transmittance_v1.fits" + att_telescope-1_cmos_obfilter: "attenuation-data/foxsi4_telescope-1_BASIC_optical_blocking_filter_transmittance_v1.fits" + att_telescope-0_cmos_prefilter: "attenuation-data/foxsi4_telescope-0_BASIC_attenuation_filter_transmittance_v1.fits" + att_telescope-1_cmos_prefilter: "attenuation-data/foxsi4_telescope-1_BASIC_attenuation_filter_transmittance_v1.fits" + att_foxsi4_atmosphere: "attenuation-data/FOXSI4_atmospheric_transmission_v1.fits" + + quantum_efficiency: + qe_cmos_telescope-0: "quantum-efficiency-data/foxsi4_telescope-0_BASIC_sensor_quantum_efficiency_v1.fits" + qe_cmos_telescope-1: "quantum-efficiency-data/foxsi4_telescope-1_BASIC_sensor_quantum_efficiency_v1.fits" # remote URL for fetching data. # can validly append any filename above to the server name to access. From 6047f408f2806993b9d2b794a8a3f27e11824df8 Mon Sep 17 00:00:00 2001 From: thanasipantazides Date: Wed, 24 Sep 2025 17:19:34 -0500 Subject: [PATCH 4/6] updating to new project structure --- .../io/fetch_response_data.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {response-tools-py/response_tools_py => response_tools}/io/fetch_response_data.py (100%) diff --git a/response-tools-py/response_tools_py/io/fetch_response_data.py b/response_tools/io/fetch_response_data.py similarity index 100% rename from response-tools-py/response_tools_py/io/fetch_response_data.py rename to response_tools/io/fetch_response_data.py From e90c0207a67c92f80213694746589efb8e426b09 Mon Sep 17 00:00:00 2001 From: Thanasi Pantazides Date: Wed, 24 Sep 2025 23:03:12 -0500 Subject: [PATCH 5/6] Correct info.yaml to include X-7 and X-8 tilt files --- response-information/info.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/response-information/info.yaml b/response-information/info.yaml index 51691b6..46b0e59 100644 --- a/response-information/info.yaml +++ b/response-information/info.yaml @@ -1,9 +1,9 @@ files: optics: eff_area_telescope-2-pan_msfc_heritage: "effective-area-data/FOXSI3_Module_X-7_EA_pan_v1.txt" - eff_area_telescope-2-tilt_msfc_heritage: "effective-area-data/FOXSI3_Module_X-7_EA_pan_v1.txt" + eff_area_telescope-2-tilt_msfc_heritage: "effective-area-data/FOXSI3_Module_X-7_EA_tilt_v1.txt" eff_area_telescope-5-pan_msfc_heritage: "effective-area-data/FOXSI3_Module_X-8_EA_pan_v1.txt" - eff_area_telescope-5-tilt_msfc_heritage: "effective-area-data/FOXSI3_Module_X-8_EA_pan_v1.txt" + eff_area_telescope-5-tilt_msfc_heritage: "effective-area-data/FOXSI3_Module_X-8_EA_tilt_v1.txt" eff_area_msfc_hi_res: "effective-area-data/FOXSI4_Module_MSFC_HiRes_EA_with_models_v1.txt" eff_area_nagoya_hxt: "effective-area-data/nagoya_hxt_onaxis_measurement_v1.txt" eff_area_nagoya_sxt: "effective-area-data/nagoya_sxt_onaxis_measurement_v1.txt" From 556bef90c001590e40724cbb85b0754ee4a0cc65 Mon Sep 17 00:00:00 2001 From: Thanasi Pantazides Date: Wed, 24 Sep 2025 23:03:47 -0500 Subject: [PATCH 6/6] Add function to download all files required by info.yaml --- response_tools/io/fetch_response_data.py | 161 ++++++++++++++--------- setup.py | 2 + 2 files changed, 104 insertions(+), 59 deletions(-) diff --git a/response_tools/io/fetch_response_data.py b/response_tools/io/fetch_response_data.py index e76a043..7c67718 100644 --- a/response_tools/io/fetch_response_data.py +++ b/response_tools/io/fetch_response_data.py @@ -2,27 +2,18 @@ Script to download data from FOXSI server. """ -import urllib.request import os, cmd, sys +# from pathlib import PureWindowsPath, PurePosixPath +import urllib.request +from urllib.parse import urljoin +from tqdm import tqdm from enum import Enum +from response_tools.io.load_yaml import load_response_context import inquirer -from bs4 import BeautifulSoup as bs -from astropy.io import fits - import pprint -remote = "http://foxsi.space.umn.edu/data/response/response-components/" local_prefix = 'response-information' -area_files = [ - "http://foxsi.space.umn.edu/data/response/response-components/effective-area-data/foxsi4_telescope-0_BASIC_mirror_effective_area_v1.fits", - "http://foxsi.space.umn.edu/data/response/response-components/effective-area-data/foxsi4_telescope-1_BASIC_mirror_effective_area_v1.fits", - "http://foxsi.space.umn.edu/data/response/response-components/effective-area-data/nagoya_hxt_onaxis_measurement_v1.txt", - "http://foxsi.space.umn.edu/data/response/response-components/effective-area-data/nagoya_sxt_onaxis_measurement_v1.txt", - "http://foxsi.space.umn.edu/data/response/response-components/effective-area-data/FOXSI4_Module_MSFC_HiRes_EA_with_models_v1.txt", - "http://foxsi.space.umn.edu/data/response/response-components/effective-area-data/foxsi4_telescope-0_BASIC_TELESCOPE_RESPONSE_V25APR13.fits", - "http://foxsi.space.umn.edu/data/response/response-components/effective-area-data/foxsi4_telescope-1_BASIC_TELESCOPE_RESPONSE_V25APR13.fits" -] ignore_urls = ['@eaDir/'] @@ -131,51 +122,103 @@ def _prompt_file(self): print(answers["file"]) -if __name__ == "__main__": - DownloadPrompt() - # pprint.pprint(answers) - - # if answers["query"] == "" - - - # for file in area_files: - # url_suffix = os.path.join(os.path.split(file)[-2], os.path.split(file)[-1]) - # url_dir_prefix, url_file = os.path.split(file) - # _, url_dir = os.path.split(url_dir_prefix) - - # local_dest = os.path.abspath(os.path.join(__file__, '..', '..', '..', '..', local_prefix, url_dir, url_file)) - # if os.path.isfile(local_dest): - # print_red('\talready have local copy of ' + url_file + ', skipping.') - # continue - # local_name, local_ext = os.path.splitext(local_dest) - # local_ver = local_name.rsplit('v', 1) +def green_str(text:str): + return "\033[92m" + text + "\033[0m" + +def foxsi4_download_required(replace_existing=False, verbose=False): + """Download all response component files specified in `response-information/info.yaml`. + + Download data products from a remote server to the local filesystem. Retrieves server + URL and all local paths for saving data from a config file: + `response-tools/response-information/info.yaml`. All downloaded response data will be + saved under `response-tools/response-information`. + + Parameters + ---------- + replace_existing : `bool` + Whether to replace local files with newer versions, if newer versions are + downloaded. Currently throws `NotImplementedError`. + + verbose : `bool` + Toggle for printing verbosely. If `True`, download progress indicators and + filenames are displayed. If `False`, nothing is printed at all. + + Returns + ------- + : `downloaded` + A dict of downloaded data. Keys are the same file identifiers from the YAML + source. Values are the absolute paths on the local filesystem to the downloaded + file. Files which were already existed in the local filesystem (required no + downloaded) are not included in the return value. + """ + + if replace_existing == True: + raise NotImplementedError("No support yet for replacement of old file versions.") + + # print if the verbose flag is set: + def verbose_print(*something): + if verbose: + print(*something) + + req = load_response_context() + server_url = req["remote_server"] + + # for urllib.parse.urljoin to work correctly, server path prefix must end in `/`: + if server_url[-1] != "/": + server_url += "/" + + # directory on local filesystem for saving data: + local_info_dir = os.path.abspath(os.path.join(__file__, "..", "..", "..", "response-information")) + verbose_print("Retrieving response products from:", green_str(server_url)) + verbose_print("Saving response products to:", green_str(local_info_dir)) + + # record which files already exist on-disk (don't waste time downloading): + existing_files = [] + for r,_,fs in os.walk(local_info_dir): + for f in fs: + existing_files.append(os.path.join(r,f)) + + desired_files = [] # list of the files to download + destination_path = [] # local path to save them to + source_name = [] # identifier of the file (YAML key) + do_get = [] # flag whether to download (if the file already exists locally) + + for comp_name in req["files"].keys(): + for f_name, suffix in req["files"][comp_name].items(): + desired_files.append(urljoin(server_url, suffix)) + dest = os.path.join(local_info_dir, suffix) + destination_path.append(dest) + source_name.append(f_name) + + if os.path.exists(dest): + do_get.append(False) + else: + do_get.append(True) + + downloaded = {} + if any(do_get): + verbose_print("Retrieving files...") + for (k, f) in enumerate(tqdm(desired_files, disable=not verbose)): + if do_get[k]: + try: + # create the folders along the save path, if needed + os.makedirs(os.path.dirname(destination_path[k])) + except: + pass + + # download the file: + fname, head = urllib.request.urlretrieve(f, destination_path[k]) + # record the identifier and path of the downloaded file: + downloaded[source_name[k]] = fname + if verbose: + tqdm.write("Downloaded " + green_str(os.path.basename(fname))) + else: + verbose_print("Found nothing new to download") + return downloaded - # print('found new file: ' + local_dest) - # print_green('\tfound version: ' + local_ver[-1]) - # print_green('\t' + local_dest) - - # fname, head = urllib.request.urlretrieve(file, local_dest) - # # d = f.read() - - # if os.path.splitext(fname) == '.fits': - # with fits.open(fname) as hdul: - # print(hdul.info()) - # elif os.path.splitext(fname) == '.csv': - # print('a csv...') - - - - # f = urllib.request.urlopen(remote) - # d = f.read() - - # soup = bs(d, 'html.parser') - # print('reading page:', '\033[92m' + soup.title.string + '\033[0m ...') +if __name__ == "__main__": + # DownloadPrompt() - # links = soup.find_all('a') # select all HTML tags - # print('found links:') - # for link in links: # check each link - # h = link.get('href') # get the hyperref - # if h[-1] == '/': # find folders (end in `/`) - # if h not in ignore_urls: # ignore the ignorables - # print_green('\t' + h) + downloaded = foxsi4_download_required(verbose=True) + pprint.pprint(downloaded) \ No newline at end of file diff --git a/setup.py b/setup.py index 7c8254c..3450c0d 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,8 @@ "astropy", "pytest", "pandas", + "beautifulsoup4", + "inquirer" ], packages=setuptools.find_packages(), zip_safe=False,