diff --git a/hepdata/config.py b/hepdata/config.py index 9065b9d39..cbbc39de3 100644 --- a/hepdata/config.py +++ b/hepdata/config.py @@ -196,7 +196,7 @@ def _(x): CFG_SEARCH_RANGE_TERMS = ["recid", "publication_recid", "inspire_id"] # Possible terms used to OpenSearch API range searches CFG_CONVERTER_URL = 'https://converter.hepdata.net' -CFG_SUPPORTED_FORMATS = ['yaml', 'root', 'csv', 'yoda', 'yoda1', 'original'] +CFG_SUPPORTED_FORMATS = ['yaml', 'root', 'csv', 'yoda', 'yoda1', 'yoda.h5', 'original'] CFG_CONVERTER_TIMEOUT = 220 # timeout in seconds CFG_TMPDIR = tempfile.gettempdir() diff --git a/hepdata/ext/opensearch/document_enhancers.py b/hepdata/ext/opensearch/document_enhancers.py index 52d68fdf5..a2e217cd6 100644 --- a/hepdata/ext/opensearch/document_enhancers.py +++ b/hepdata/ext/opensearch/document_enhancers.py @@ -37,7 +37,7 @@ from hepdata.modules.submission.models import DataSubmission from hepdata.utils.miscellaneous import get_resource_data -FORMATS = ['json', 'root', 'yaml', 'csv', 'yoda'] +FORMATS = ['json', 'root', 'yaml', 'csv', 'yoda', 'yoda.h5'] logging.basicConfig() log = logging.getLogger(__name__) diff --git a/hepdata/modules/converter/views.py b/hepdata/modules/converter/views.py index 8b808deea..fd3c0654b 100644 --- a/hepdata/modules/converter/views.py +++ b/hepdata/modules/converter/views.py @@ -60,7 +60,7 @@ @blueprint.route(f'/submission//') @blueprint.route(f'/submission///') -@blueprint.route('/submission////') +@blueprint.route('/submission////') def download_submission_with_inspire_id(*args, **kwargs): """ Gets the submission file and either serves it back directly from YAML, or converts it @@ -71,7 +71,7 @@ def download_submission_with_inspire_id(*args, **kwargs): :param inspire_id: inspire id :param version: version of submission to export. If absent, returns the latest. - :param file_format: json, yaml, csv, root, yoda, yoda1 or original + :param file_format: json, yaml, csv, root, yoda, yoda1, yoda.h5 or original :param rivet: Rivet analysis name to override default written in YODA export :return: download_submission """ @@ -115,7 +115,7 @@ def download_submission_with_inspire_id(*args, **kwargs): @blueprint.route(f'/submission//') @blueprint.route(f'/submission///') -@blueprint.route('/submission////') +@blueprint.route('/submission////') def download_submission_with_recid(*args, **kwargs): """ Gets the submission file and either serves it back directly from YAML, or converts it @@ -126,7 +126,7 @@ def download_submission_with_recid(*args, **kwargs): :param recid: submissions recid :param version: version of submission to export. If absent, returns the latest. - :param file_format: json, yaml, csv, root, yoda, yoda1 or original + :param file_format: json, yaml, csv, root, yoda, yoda1, yoda.h5 or original :param rivet: Rivet analysis name to override default written in YODA export :return: download_submission """ @@ -160,7 +160,7 @@ def download_submission(submission, file_format, offline=False, force=False, riv for other formats. :param submission: HEPSubmission - :param file_format: json, yaml, csv, root, yoda, yoda1 or original + :param file_format: json, yaml, csv, root, yoda, yoda1, yoda.h5 or original :param offline: offline creation of the conversion when a record is finalised :param force: force recreation of the conversion :param rivet_analysis_name: Rivet analysis name to override default written in YODA export @@ -188,7 +188,8 @@ def download_submission(submission, file_format, offline=False, force=False, riv if file_format == 'original': file_format_and_extension = os.path.splitext(data_filepath)[1] else: - file_format_and_extension = '-{0}.tar.gz'.format(file_format) + file_format_dashed = file_format.replace('.', '-') + file_format_and_extension = '-{0}.tar.gz'.format(file_format_dashed) output_file = 'HEPData-{0}-v{1}{2}'.format(file_identifier, submission.version, file_format_and_extension) @@ -219,10 +220,11 @@ def download_submission(submission, file_format, offline=False, force=False, riv print('File created at {0}'.format(output_path)) return + file_format_dashed = file_format.replace('.', '-') converter_options = { 'input_format': 'yaml', 'output_format': file_format, - 'filename': 'HEPData-{0}-v{1}-{2}'.format(file_identifier, submission.version, file_format), + 'filename': 'HEPData-{0}-v{1}-{2}'.format(file_identifier, submission.version, file_format_dashed), 'validator_schema_version': '0.1.0', } @@ -253,7 +255,7 @@ def download_submission(submission, file_format, offline=False, force=False, riv @blueprint.route(f'/table///') @blueprint.route(f'/table////') -@blueprint.route('/table/////') +@blueprint.route('/table/////') def download_data_table_by_inspire_id(*args, **kwargs): """ Downloads the latest data file given the url ``/download/submission/ins1283842/Table 1/yaml`` or @@ -324,7 +326,7 @@ def download_data_table_by_inspire_id(*args, **kwargs): @blueprint.route(f'/table///') @blueprint.route(f'/table////') -@blueprint.route('/table/////') +@blueprint.route('/table/////') def download_data_table_by_recid(*args, **kwargs): """ Record ID download. diff --git a/hepdata/modules/records/utils/data_processing_utils.py b/hepdata/modules/records/utils/data_processing_utils.py index d8c492049..06991135c 100644 --- a/hepdata/modules/records/utils/data_processing_utils.py +++ b/hepdata/modules/records/utils/data_processing_utils.py @@ -311,18 +311,9 @@ def process_ctx(ctx, light_mode=False): _cleaned_table_name = data_table['name'].replace('%', '%25').replace('\\', '%5C') - data_table['data'] = { - 'json': '{0}/download/table/{1}/{2}/json'.format( - site_url, _recid, _cleaned_table_name), - 'root': '{0}/download/table/{1}/{2}/root'.format( - site_url, _recid, _cleaned_table_name), - 'csv': '{0}/download/table/{1}/{2}/csv'.format( - site_url, _recid, _cleaned_table_name), - 'yoda': '{0}/download/table/{1}/{2}/yoda'.format( - site_url, _recid, _cleaned_table_name), - 'yoda1': '{0}/download/table/{1}/{2}/yoda1'.format( - site_url, _recid, _cleaned_table_name), - 'yaml': '{0}/download/table/{1}/{2}/yaml'.format( - site_url, _recid, _cleaned_table_name)} + data_table['data'] = {} + for file_format in ['json', 'root', 'csv', 'yoda', 'yoda1', 'yoda.h5', 'yaml']: + data_table['data'][file_format] = '{0}/download/table/{1}/{2}/{3}'.format( + site_url, _recid, _cleaned_table_name, file_format) return ctx diff --git a/hepdata/modules/theme/templates/hepdata_theme/pages/formats.html b/hepdata/modules/theme/templates/hepdata_theme/pages/formats.html index 054d8b26b..d3fd2d3f3 100644 --- a/hepdata/modules/theme/templates/hepdata_theme/pages/formats.html +++ b/hepdata/modules/theme/templates/hepdata_theme/pages/formats.html @@ -109,6 +109,7 @@

Detailed record

toolkit. Again, the appropriate YODA object is written according to the number of independent variables in a table. The default yoda output is now in the YODA2 format, but there is still an option yoda1 to output the legacy YODA1 format. + Alternatively, the output can be given in the yoda.h5 format, designed for parallel computing.

@@ -123,6 +124,7 @@

Detailed record

  • ?format=root
  • ?format=yoda
  • ?format=yoda1
    • +
  • ?format=yoda.h5

    • Optional parameters can also be added (separated by a & symbol):

      @@ -140,7 +142,7 @@

      Detailed record

      the data tables from the response.
    • - rivet=ALICE_2016_I1419244: when using format=yoda or format=yoda1, specify the desired + rivet=ALICE_2016_I1419244: when using format=yoda, format=yoda1, or format=yoda.h5, specify the desired Rivet analysis name to be written in the YODA files if it does not match the automatically generated name.
    diff --git a/tests/e2e/test_general.py b/tests/e2e/test_general.py index dfe919eb4..edf45eeab 100644 --- a/tests/e2e/test_general.py +++ b/tests/e2e/test_general.py @@ -269,7 +269,8 @@ def test_accept_headers(app, live_server, e2e_identifiers): {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/root', 'description': 'ROOT file', 'encodingFormat': 'https://root.cern'}, {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/yaml', 'description': 'YAML file', 'encodingFormat': 'https://yaml.org'}, {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/csv', 'description': 'CSV file', 'encodingFormat': 'text/csv'}, - {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/yoda', 'description': 'YODA file', 'encodingFormat': 'https://yoda.hepforge.org'} + {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/yoda', 'description': 'YODA file', 'encodingFormat': 'https://yoda.hepforge.org'}, + {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/yoda.h5', 'description': 'YODA-H5 file', 'encodingFormat': 'https://yoda.hepforge.org'} ] # Data resource landing page (use last submission, which has resources) diff --git a/tests/records_test.py b/tests/records_test.py index d701ae69f..ebd44f57f 100644 --- a/tests/records_test.py +++ b/tests/records_test.py @@ -919,6 +919,12 @@ def test_get_json_ld(app, load_default_data, identifiers): 'contentUrl': f'http://localhost:5000/download/table/2/yoda', 'description': 'YODA file', 'encodingFormat': 'https://yoda.hepforge.org' + }, + { + '@type': 'DataDownload', + 'contentUrl': f'http://localhost:5000/download/table/2/yoda.h5', + 'description': 'YODA-H5 file', + 'encodingFormat': 'https://yoda.hepforge.org' } ] assert table_data['includedInDataCatalog'] == {