Add support for yoda-H5

hepdata/config.py

@@ -196,7 +196,7 @@ def _(x):
 CFG_SEARCH_RANGE_TERMS = ["recid", "publication_recid", "inspire_id"]  # Possible terms used to OpenSearch API range searches
 
 CFG_CONVERTER_URL = 'https://converter.hepdata.net'
-CFG_SUPPORTED_FORMATS = ['yaml', 'root', 'csv', 'yoda', 'yoda1', 'original']
+CFG_SUPPORTED_FORMATS = ['yaml', 'root', 'csv', 'yoda', 'yoda1', 'yoda.h5', 'original']
 CFG_CONVERTER_TIMEOUT = 220  # timeout in seconds
 
 CFG_TMPDIR = tempfile.gettempdir()

hepdata/ext/opensearch/document_enhancers.py

@@ -37,7 +37,7 @@
 from hepdata.modules.submission.models import DataSubmission
 from hepdata.utils.miscellaneous import get_resource_data
 
-FORMATS = ['json', 'root', 'yaml', 'csv', 'yoda']
+FORMATS = ['json', 'root', 'yaml', 'csv', 'yoda', 'yoda.h5']
 
 logging.basicConfig()
 log = logging.getLogger(__name__)

hepdata/modules/converter/views.py

@@ -60,7 +60,7 @@
 
 @blueprint.route(f'/submission/<inspire_id>/<any({FORMATS}):file_format>')
 @blueprint.route(f'/submission/<inspire_id>/<int:version>/<any({FORMATS}):file_format>')
-@blueprint.route('/submission/<inspire_id>/<int:version>/<any(yoda,yoda1):file_format>/<rivet>')
+@blueprint.route('/submission/<inspire_id>/<int:version>/<any(yoda,yoda1,yoda.h5):file_format>/<rivet>')
 def download_submission_with_inspire_id(*args, **kwargs):
     """
     Gets the submission file and either serves it back directly from YAML, or converts it
@@ -71,7 +71,7 @@ def download_submission_with_inspire_id(*args, **kwargs):
 
     :param inspire_id: inspire id
     :param version: version of submission to export. If absent, returns the latest.
-    :param file_format: json, yaml, csv, root, yoda, yoda1 or original
+    :param file_format: json, yaml, csv, root, yoda, yoda1, yoda.h5 or original
     :param rivet: Rivet analysis name to override default written in YODA export
     :return: download_submission
     """
@@ -115,7 +115,7 @@ def download_submission_with_inspire_id(*args, **kwargs):
 
 @blueprint.route(f'/submission/<int:recid>/<any({FORMATS}):file_format>')
 @blueprint.route(f'/submission/<int:recid>/<int:version>/<any({FORMATS}):file_format>')
-@blueprint.route('/submission/<int:recid>/<int:version>/<any(yoda,yoda1):file_format>/<rivet>')
+@blueprint.route('/submission/<int:recid>/<int:version>/<any(yoda,yoda1,yoda.h5):file_format>/<rivet>')
 def download_submission_with_recid(*args, **kwargs):
     """
     Gets the submission file and either serves it back directly from YAML, or converts it
@@ -126,7 +126,7 @@ def download_submission_with_recid(*args, **kwargs):
 
     :param recid: submissions recid
     :param version: version of submission to export. If absent, returns the latest.
-    :param file_format: json, yaml, csv, root, yoda, yoda1 or original
+    :param file_format: json, yaml, csv, root, yoda, yoda1, yoda.h5 or original
     :param rivet: Rivet analysis name to override default written in YODA export
     :return: download_submission
     """
@@ -160,7 +160,7 @@ def download_submission(submission, file_format, offline=False, force=False, riv
     for other formats.
 
     :param submission: HEPSubmission
-    :param file_format: json, yaml, csv, root, yoda, yoda1 or original
+    :param file_format: json, yaml, csv, root, yoda, yoda1, yoda.h5 or original
     :param offline: offline creation of the conversion when a record is finalised
     :param force: force recreation of the conversion
     :param rivet_analysis_name: Rivet analysis name to override default written in YODA export
@@ -188,7 +188,8 @@ def download_submission(submission, file_format, offline=False, force=False, riv
     if file_format == 'original':
         file_format_and_extension = os.path.splitext(data_filepath)[1]
     else:
-        file_format_and_extension = '-{0}.tar.gz'.format(file_format)
+        file_format_dashed = file_format.replace('.', '-')
+        file_format_and_extension = '-{0}.tar.gz'.format(file_format_dashed)
 
     output_file = 'HEPData-{0}-v{1}{2}'.format(file_identifier, submission.version, file_format_and_extension)
 
@@ -219,10 +220,11 @@ def download_submission(submission, file_format, offline=False, force=False, riv
             print('File created at {0}'.format(output_path))
             return
 
+    file_format_dashed = file_format.replace('.', '-')
     converter_options = {
         'input_format': 'yaml',
         'output_format': file_format,
-        'filename': 'HEPData-{0}-v{1}-{2}'.format(file_identifier, submission.version, file_format),
+        'filename': 'HEPData-{0}-v{1}-{2}'.format(file_identifier, submission.version, file_format_dashed),
         'validator_schema_version': '0.1.0',
     }
 
@@ -253,7 +255,7 @@ def download_submission(submission, file_format, offline=False, force=False, riv
 
 @blueprint.route(f'/table/<inspire_id>/<path:table_name>/<any({FORMATS}):file_format>')
 @blueprint.route(f'/table/<inspire_id>/<path:table_name>/<int:version>/<any({FORMATS}):file_format>')
-@blueprint.route('/table/<inspire_id>/<path:table_name>/<int:version>/<any(yoda,yoda1):file_format>/<rivet>')
+@blueprint.route('/table/<inspire_id>/<path:table_name>/<int:version>/<any(yoda,yoda1,yoda.h5):file_format>/<rivet>')
 def download_data_table_by_inspire_id(*args, **kwargs):
     """
     Downloads the latest data file given the url ``/download/submission/ins1283842/Table 1/yaml`` or
@@ -324,7 +326,7 @@ def download_data_table_by_inspire_id(*args, **kwargs):
 
 @blueprint.route(f'/table/<int:recid>/<path:table_name>/<any({FORMATS}):file_format>')
 @blueprint.route(f'/table/<int:recid>/<path:table_name>/<int:version>/<any({FORMATS}):file_format>')
-@blueprint.route('/table/<int:recid>/<path:table_name>/<int:version>/<any(yoda,yoda1):file_format>/<rivet>')
+@blueprint.route('/table/<int:recid>/<path:table_name>/<int:version>/<any(yoda,yoda1,yoda.h5):file_format>/<rivet>')
 def download_data_table_by_recid(*args, **kwargs):
     """
     Record ID download.

hepdata/modules/records/utils/data_processing_utils.py

@@ -311,18 +311,9 @@ def process_ctx(ctx, light_mode=False):
 
                 _cleaned_table_name = data_table['name'].replace('%', '%25').replace('\\', '%5C')
 
-                data_table['data'] = {
-                    'json': '{0}/download/table/{1}/{2}/json'.format(
-                        site_url, _recid, _cleaned_table_name),
-                    'root': '{0}/download/table/{1}/{2}/root'.format(
-                        site_url, _recid, _cleaned_table_name),
-                    'csv': '{0}/download/table/{1}/{2}/csv'.format(
-                        site_url, _recid, _cleaned_table_name),
-                    'yoda': '{0}/download/table/{1}/{2}/yoda'.format(
-                        site_url, _recid, _cleaned_table_name),
-                    'yoda1': '{0}/download/table/{1}/{2}/yoda1'.format(
-                        site_url, _recid, _cleaned_table_name),
-                    'yaml': '{0}/download/table/{1}/{2}/yaml'.format(
-                        site_url, _recid, _cleaned_table_name)}
+                data_table['data'] = {}
+                for file_format in ['json', 'root', 'csv', 'yoda', 'yoda1', 'yoda.h5', 'yaml']:
+                    data_table['data'][file_format] = '{0}/download/table/{1}/{2}/{3}'.format(
+                        site_url, _recid, _cleaned_table_name, file_format)
 
     return ctx

hepdata/modules/theme/templates/hepdata_theme/pages/formats.html

@@ -109,6 +109,7 @@ <h3>Detailed record</h3>
                     toolkit. Again, the appropriate <a href="https://yoda.hepforge.org" target="_blank" style="text-decoration: underline">YODA</a> object
                     is written according to the number of independent variables in a table. The default <code>yoda</code> output is now
                     in the YODA2 format, but there is still an option <code>yoda1</code> to output the legacy YODA1 format.
+                    Alternatively, the output can be given in the <code>yoda.h5</code> format, designed for parallel computing.
                   </li>
                 </ul>
                 <p>
@@ -123,6 +124,7 @@ <h3>Detailed record</h3>
                   <li><code>?format=root</code></li>
                   <li><code>?format=yoda</code></li>
                   <li><code>?format=yoda1</code></li>
+                  <li><code>?format=yoda.h5</code></li>
                 </ul>
                 <p>Optional parameters can also be added (separated by a <code>&</code> symbol):</p>
                 <ul>
@@ -140,7 +142,7 @@ <h3>Detailed record</h3>
                     the data tables from the response.
                   </li>
                   <li>
-                    <code>rivet=ALICE_2016_I1419244</code>: when using <code>format=yoda</code> or <code>format=yoda1</code>, specify the desired
+                    <code>rivet=ALICE_2016_I1419244</code>: when using <code>format=yoda</code>, <code>format=yoda1</code>, or <code>format=yoda.h5</code>, specify the desired
                     Rivet analysis name to be written in the YODA files if it does not match the automatically generated name.
                   </li>
                 </ul>

tests/e2e/test_general.py

@@ -269,7 +269,8 @@ def test_accept_headers(app, live_server, e2e_identifiers):
         {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/root', 'description': 'ROOT file', 'encodingFormat': 'https://root.cern'},
         {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/yaml', 'description': 'YAML file', 'encodingFormat': 'https://yaml.org'},
         {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/csv', 'description': 'CSV file', 'encodingFormat': 'text/csv'},
-        {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/yoda', 'description': 'YODA file', 'encodingFormat': 'https://yoda.hepforge.org'}
+        {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/yoda', 'description': 'YODA file', 'encodingFormat': 'https://yoda.hepforge.org'},
+        {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/yoda.h5', 'description': 'YODA-H5 file', 'encodingFormat': 'https://yoda.hepforge.org'}
     ]
 
     # Data resource landing page (use last submission, which has resources)

tests/records_test.py

@@ -919,6 +919,12 @@ def test_get_json_ld(app, load_default_data, identifiers):
             'contentUrl': f'http://localhost:5000/download/table/2/yoda',
             'description': 'YODA file',
             'encodingFormat': 'https://yoda.hepforge.org'
+        },
+        {
+            '@type': 'DataDownload',
+            'contentUrl': f'http://localhost:5000/download/table/2/yoda.h5',
+            'description': 'YODA-H5 file',
+            'encodingFormat': 'https://yoda.hepforge.org'
         }
     ]
     assert table_data['includedInDataCatalog'] == {