From 94d9fdb27272c3d04f887412c5110cdb9959e568 Mon Sep 17 00:00:00 2001 From: Martin Habedank Date: Wed, 21 May 2025 10:10:16 +0100 Subject: [PATCH 1/5] allow download as yoda.h5 format --- hepdata/config.py | 2 +- hepdata/ext/opensearch/document_enhancers.py | 2 +- hepdata/modules/converter/views.py | 14 +++++++------- .../modules/records/utils/data_processing_utils.py | 6 ++++++ .../templates/hepdata_theme/pages/formats.html | 4 +++- 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/hepdata/config.py b/hepdata/config.py index 9065b9d39..cbbc39de3 100644 --- a/hepdata/config.py +++ b/hepdata/config.py @@ -196,7 +196,7 @@ def _(x): CFG_SEARCH_RANGE_TERMS = ["recid", "publication_recid", "inspire_id"] # Possible terms used to OpenSearch API range searches CFG_CONVERTER_URL = 'https://converter.hepdata.net' -CFG_SUPPORTED_FORMATS = ['yaml', 'root', 'csv', 'yoda', 'yoda1', 'original'] +CFG_SUPPORTED_FORMATS = ['yaml', 'root', 'csv', 'yoda', 'yoda1', 'yoda.h5', 'original'] CFG_CONVERTER_TIMEOUT = 220 # timeout in seconds CFG_TMPDIR = tempfile.gettempdir() diff --git a/hepdata/ext/opensearch/document_enhancers.py b/hepdata/ext/opensearch/document_enhancers.py index 52d68fdf5..a2e217cd6 100644 --- a/hepdata/ext/opensearch/document_enhancers.py +++ b/hepdata/ext/opensearch/document_enhancers.py @@ -37,7 +37,7 @@ from hepdata.modules.submission.models import DataSubmission from hepdata.utils.miscellaneous import get_resource_data -FORMATS = ['json', 'root', 'yaml', 'csv', 'yoda'] +FORMATS = ['json', 'root', 'yaml', 'csv', 'yoda', 'yoda.h5'] logging.basicConfig() log = logging.getLogger(__name__) diff --git a/hepdata/modules/converter/views.py b/hepdata/modules/converter/views.py index 8b808deea..be6dc9cda 100644 --- a/hepdata/modules/converter/views.py +++ b/hepdata/modules/converter/views.py @@ -60,7 +60,7 @@ @blueprint.route(f'/submission//') @blueprint.route(f'/submission///') -@blueprint.route('/submission////') +@blueprint.route('/submission////') def download_submission_with_inspire_id(*args, **kwargs): """ Gets the submission file and either serves it back directly from YAML, or converts it @@ -71,7 +71,7 @@ def download_submission_with_inspire_id(*args, **kwargs): :param inspire_id: inspire id :param version: version of submission to export. If absent, returns the latest. - :param file_format: json, yaml, csv, root, yoda, yoda1 or original + :param file_format: json, yaml, csv, root, yoda, yoda1, yoda.h5 or original :param rivet: Rivet analysis name to override default written in YODA export :return: download_submission """ @@ -115,7 +115,7 @@ def download_submission_with_inspire_id(*args, **kwargs): @blueprint.route(f'/submission//') @blueprint.route(f'/submission///') -@blueprint.route('/submission////') +@blueprint.route('/submission////') def download_submission_with_recid(*args, **kwargs): """ Gets the submission file and either serves it back directly from YAML, or converts it @@ -126,7 +126,7 @@ def download_submission_with_recid(*args, **kwargs): :param recid: submissions recid :param version: version of submission to export. If absent, returns the latest. - :param file_format: json, yaml, csv, root, yoda, yoda1 or original + :param file_format: json, yaml, csv, root, yoda, yoda1, yoda.h5 or original :param rivet: Rivet analysis name to override default written in YODA export :return: download_submission """ @@ -160,7 +160,7 @@ def download_submission(submission, file_format, offline=False, force=False, riv for other formats. :param submission: HEPSubmission - :param file_format: json, yaml, csv, root, yoda, yoda1 or original + :param file_format: json, yaml, csv, root, yoda, yoda1, yoda.h5 or original :param offline: offline creation of the conversion when a record is finalised :param force: force recreation of the conversion :param rivet_analysis_name: Rivet analysis name to override default written in YODA export @@ -253,7 +253,7 @@ def download_submission(submission, file_format, offline=False, force=False, riv @blueprint.route(f'/table///') @blueprint.route(f'/table////') -@blueprint.route('/table/////') +@blueprint.route('/table/////') def download_data_table_by_inspire_id(*args, **kwargs): """ Downloads the latest data file given the url ``/download/submission/ins1283842/Table 1/yaml`` or @@ -324,7 +324,7 @@ def download_data_table_by_inspire_id(*args, **kwargs): @blueprint.route(f'/table///') @blueprint.route(f'/table////') -@blueprint.route('/table/////') +@blueprint.route('/table/////') def download_data_table_by_recid(*args, **kwargs): """ Record ID download. diff --git a/hepdata/modules/records/utils/data_processing_utils.py b/hepdata/modules/records/utils/data_processing_utils.py index d8c492049..4f3279174 100644 --- a/hepdata/modules/records/utils/data_processing_utils.py +++ b/hepdata/modules/records/utils/data_processing_utils.py @@ -311,6 +311,11 @@ def process_ctx(ctx, light_mode=False): _cleaned_table_name = data_table['name'].replace('%', '%25').replace('\\', '%5C') + #data_table['data'] = {} + #for format in ['json', 'root', 'csv', 'yoda', 'yoda1', 'yoda.h5', 'yaml']: + # data_table[format] = '{0}/download/table/{1}/{2}/{4}'.format( + # site_url, _recid, _cleaned_table_name, format) + data_table['data'] = { 'json': '{0}/download/table/{1}/{2}/json'.format( site_url, _recid, _cleaned_table_name), @@ -325,4 +330,5 @@ def process_ctx(ctx, light_mode=False): 'yaml': '{0}/download/table/{1}/{2}/yaml'.format( site_url, _recid, _cleaned_table_name)} + return ctx diff --git a/hepdata/modules/theme/templates/hepdata_theme/pages/formats.html b/hepdata/modules/theme/templates/hepdata_theme/pages/formats.html index 054d8b26b..70dc4c011 100644 --- a/hepdata/modules/theme/templates/hepdata_theme/pages/formats.html +++ b/hepdata/modules/theme/templates/hepdata_theme/pages/formats.html @@ -109,6 +109,7 @@

Detailed record

toolkit. Again, the appropriate YODA object is written according to the number of independent variables in a table. The default yoda output is now in the YODA2 format, but there is still an option yoda1 to output the legacy YODA1 format. + Alternatively, the output can be given in the yoda.h5 format, designed for parallel computing.

@@ -123,6 +124,7 @@

Detailed record

  • ?format=root
  • ?format=yoda
  • ?format=yoda1
    • +
  • ?format=yoda.h5

    • Optional parameters can also be added (separated by a & symbol):

      @@ -140,7 +142,7 @@

      Detailed record

      the data tables from the response.
    • - rivet=ALICE_2016_I1419244: when using format=yoda or format=yoda1, specify the desired + rivet=ALICE_2016_I1419244: when using format=yoda, format=yoda1 or format=yoda.h5, specify the desired Rivet analysis name to be written in the YODA files if it does not match the automatically generated name.
    From 14993364dd7daffd3187e2ce6d38ad1fcced1fa2 Mon Sep 17 00:00:00 2001 From: Martin Habedank Date: Wed, 21 May 2025 16:40:12 +0100 Subject: [PATCH 2/5] remove unnecessary change --- hepdata/modules/records/utils/data_processing_utils.py | 6 ------ .../theme/templates/hepdata_theme/pages/formats.html | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/hepdata/modules/records/utils/data_processing_utils.py b/hepdata/modules/records/utils/data_processing_utils.py index 4f3279174..d8c492049 100644 --- a/hepdata/modules/records/utils/data_processing_utils.py +++ b/hepdata/modules/records/utils/data_processing_utils.py @@ -311,11 +311,6 @@ def process_ctx(ctx, light_mode=False): _cleaned_table_name = data_table['name'].replace('%', '%25').replace('\\', '%5C') - #data_table['data'] = {} - #for format in ['json', 'root', 'csv', 'yoda', 'yoda1', 'yoda.h5', 'yaml']: - # data_table[format] = '{0}/download/table/{1}/{2}/{4}'.format( - # site_url, _recid, _cleaned_table_name, format) - data_table['data'] = { 'json': '{0}/download/table/{1}/{2}/json'.format( site_url, _recid, _cleaned_table_name), @@ -330,5 +325,4 @@ def process_ctx(ctx, light_mode=False): 'yaml': '{0}/download/table/{1}/{2}/yaml'.format( site_url, _recid, _cleaned_table_name)} - return ctx diff --git a/hepdata/modules/theme/templates/hepdata_theme/pages/formats.html b/hepdata/modules/theme/templates/hepdata_theme/pages/formats.html index 70dc4c011..d3fd2d3f3 100644 --- a/hepdata/modules/theme/templates/hepdata_theme/pages/formats.html +++ b/hepdata/modules/theme/templates/hepdata_theme/pages/formats.html @@ -142,7 +142,7 @@

    Detailed record

    the data tables from the response.
  • - rivet=ALICE_2016_I1419244: when using format=yoda, format=yoda1 or format=yoda.h5, specify the desired + rivet=ALICE_2016_I1419244: when using format=yoda, format=yoda1, or format=yoda.h5, specify the desired Rivet analysis name to be written in the YODA files if it does not match the automatically generated name.
    • From 83adff102bdc071fe04b95cf2a741ac67dba29c2 Mon Sep 17 00:00:00 2001 From: Martin Habedank Date: Wed, 21 May 2025 17:14:37 +0100 Subject: [PATCH 3/5] add yoda-h5 to test cases --- tests/e2e/test_general.py | 3 ++- tests/records_test.py | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/e2e/test_general.py b/tests/e2e/test_general.py index dfe919eb4..edf45eeab 100644 --- a/tests/e2e/test_general.py +++ b/tests/e2e/test_general.py @@ -269,7 +269,8 @@ def test_accept_headers(app, live_server, e2e_identifiers): {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/root', 'description': 'ROOT file', 'encodingFormat': 'https://root.cern'}, {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/yaml', 'description': 'YAML file', 'encodingFormat': 'https://yaml.org'}, {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/csv', 'description': 'CSV file', 'encodingFormat': 'text/csv'}, - {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/yoda', 'description': 'YODA file', 'encodingFormat': 'https://yoda.hepforge.org'} + {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/yoda', 'description': 'YODA file', 'encodingFormat': 'https://yoda.hepforge.org'}, + {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/yoda.h5', 'description': 'YODA-H5 file', 'encodingFormat': 'https://yoda.hepforge.org'} ] # Data resource landing page (use last submission, which has resources) diff --git a/tests/records_test.py b/tests/records_test.py index d701ae69f..ebd44f57f 100644 --- a/tests/records_test.py +++ b/tests/records_test.py @@ -919,6 +919,12 @@ def test_get_json_ld(app, load_default_data, identifiers): 'contentUrl': f'http://localhost:5000/download/table/2/yoda', 'description': 'YODA file', 'encodingFormat': 'https://yoda.hepforge.org' + }, + { + '@type': 'DataDownload', + 'contentUrl': f'http://localhost:5000/download/table/2/yoda.h5', + 'description': 'YODA-H5 file', + 'encodingFormat': 'https://yoda.hepforge.org' } ] assert table_data['includedInDataCatalog'] == { From 671021b8c35c9823a915d5ca17ae413d766d4008 Mon Sep 17 00:00:00 2001 From: Martin Habedank Date: Wed, 6 Aug 2025 14:13:43 +0100 Subject: [PATCH 4/5] compactify creation of links to output tables in JSON, add yoda.h5 to it --- .../records/utils/data_processing_utils.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/hepdata/modules/records/utils/data_processing_utils.py b/hepdata/modules/records/utils/data_processing_utils.py index d8c492049..06991135c 100644 --- a/hepdata/modules/records/utils/data_processing_utils.py +++ b/hepdata/modules/records/utils/data_processing_utils.py @@ -311,18 +311,9 @@ def process_ctx(ctx, light_mode=False): _cleaned_table_name = data_table['name'].replace('%', '%25').replace('\\', '%5C') - data_table['data'] = { - 'json': '{0}/download/table/{1}/{2}/json'.format( - site_url, _recid, _cleaned_table_name), - 'root': '{0}/download/table/{1}/{2}/root'.format( - site_url, _recid, _cleaned_table_name), - 'csv': '{0}/download/table/{1}/{2}/csv'.format( - site_url, _recid, _cleaned_table_name), - 'yoda': '{0}/download/table/{1}/{2}/yoda'.format( - site_url, _recid, _cleaned_table_name), - 'yoda1': '{0}/download/table/{1}/{2}/yoda1'.format( - site_url, _recid, _cleaned_table_name), - 'yaml': '{0}/download/table/{1}/{2}/yaml'.format( - site_url, _recid, _cleaned_table_name)} + data_table['data'] = {} + for file_format in ['json', 'root', 'csv', 'yoda', 'yoda1', 'yoda.h5', 'yaml']: + data_table['data'][file_format] = '{0}/download/table/{1}/{2}/{3}'.format( + site_url, _recid, _cleaned_table_name, file_format) return ctx From 2ce7ef28e62ca29b95025dde5516a12a81d084b7 Mon Sep 17 00:00:00 2001 From: Martin Habedank Date: Wed, 6 Aug 2025 14:35:17 +0100 Subject: [PATCH 5/5] used dashes instead of dots in file names --- hepdata/modules/converter/views.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hepdata/modules/converter/views.py b/hepdata/modules/converter/views.py index be6dc9cda..fd3c0654b 100644 --- a/hepdata/modules/converter/views.py +++ b/hepdata/modules/converter/views.py @@ -188,7 +188,8 @@ def download_submission(submission, file_format, offline=False, force=False, riv if file_format == 'original': file_format_and_extension = os.path.splitext(data_filepath)[1] else: - file_format_and_extension = '-{0}.tar.gz'.format(file_format) + file_format_dashed = file_format.replace('.', '-') + file_format_and_extension = '-{0}.tar.gz'.format(file_format_dashed) output_file = 'HEPData-{0}-v{1}{2}'.format(file_identifier, submission.version, file_format_and_extension) @@ -219,10 +220,11 @@ def download_submission(submission, file_format, offline=False, force=False, riv print('File created at {0}'.format(output_path)) return + file_format_dashed = file_format.replace('.', '-') converter_options = { 'input_format': 'yaml', 'output_format': file_format, - 'filename': 'HEPData-{0}-v{1}-{2}'.format(file_identifier, submission.version, file_format), + 'filename': 'HEPData-{0}-v{1}-{2}'.format(file_identifier, submission.version, file_format_dashed), 'validator_schema_version': '0.1.0', }