Add support for yoda-H5

CONTRIBUTING.rst

@@ -80,7 +80,12 @@ Ready to contribute? Here's how to set up ``hepdata`` for local development.
       $ git commit -s -m "Your detailed description of your changes."
       $ git push origin name-of-your-bugfix-or-feature
 
-7. Submit a pull request through the GitHub website, perhaps initially as a
+7. The continuous integration (CI) run via GitHub Actions requires access to secrets only available to the
+https://github.com/HEPData/hepdata repository, not to forks. Therefore, pull requests should be made only
+from branches of this repository, not from forks. You can request Write access to the repository by
+sending an email to ``info@hepdata.net``.
+
+8. Submit a pull request through the GitHub website, perhaps initially as a
 `draft pull request <https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests#draft-pull-requests>`_
 until you make sure that all checks have passed.
 

hepdata/config.py

@@ -196,7 +196,7 @@ def _(x):
 CFG_SEARCH_RANGE_TERMS = ["recid", "publication_recid", "inspire_id"]  # Possible terms used to OpenSearch API range searches
 
 CFG_CONVERTER_URL = 'https://converter.hepdata.net'
-CFG_SUPPORTED_FORMATS = ['yaml', 'root', 'csv', 'yoda', 'yoda1', 'original']
+CFG_SUPPORTED_FORMATS = ['yaml', 'root', 'csv', 'yoda', 'yoda1', 'yoda.h5', 'original']
 CFG_CONVERTER_TIMEOUT = 220  # timeout in seconds
 
 CFG_TMPDIR = tempfile.gettempdir()

hepdata/ext/opensearch/document_enhancers.py

@@ -37,7 +37,7 @@
 from hepdata.modules.submission.models import DataSubmission
 from hepdata.utils.miscellaneous import get_resource_data
 
-FORMATS = ['json', 'root', 'yaml', 'csv', 'yoda']
+FORMATS = ['json', 'root', 'yaml', 'csv', 'yoda', 'yoda.h5']
 
 logging.basicConfig()
 log = logging.getLogger(__name__)
@@ -55,7 +55,7 @@ def add_data_submission_urls(doc):
     doc['access_urls'] = {'links': {}}
 
     for format in FORMATS:
-        doc['access_urls']['links'][format] = '{0}/download/submission/ins{1}/{2}/{3}'.format(
+        doc['access_urls']['links'][format.replace('.', '-')] = '{0}/download/submission/ins{1}/{2}/{3}'.format(
             current_app.config.get('SITE_URL', 'https://www.hepdata.net'),
             doc['inspire_id'], doc['version'] if 'version' in doc else 1,
             format)
@@ -67,7 +67,7 @@ def add_data_table_urls(doc):
 
         _cleaned_table_name = doc['title'].replace('%', '%25').replace('\\', '%5C')
 
-        doc['access_urls']['links'][format] = '{0}/download/table/ins{1}/{2}/{3}'.format(
+        doc['access_urls']['links'][format.replace('.', '-')] = '{0}/download/table/ins{1}/{2}/{3}'.format(
             current_app.config.get('SITE_URL', 'https://www.hepdata.net'),
             doc['inspire_id'], _cleaned_table_name, format)
 

hepdata/modules/converter/views.py

@@ -60,7 +60,7 @@
 
 @blueprint.route(f'/submission/<inspire_id>/<any({FORMATS}):file_format>')
 @blueprint.route(f'/submission/<inspire_id>/<int:version>/<any({FORMATS}):file_format>')
-@blueprint.route('/submission/<inspire_id>/<int:version>/<any(yoda,yoda1):file_format>/<rivet>')
+@blueprint.route('/submission/<inspire_id>/<int:version>/<any(yoda,yoda1,yoda.h5):file_format>/<rivet>')
 def download_submission_with_inspire_id(*args, **kwargs):
     """
     Gets the submission file and either serves it back directly from YAML, or converts it
@@ -71,7 +71,7 @@ def download_submission_with_inspire_id(*args, **kwargs):
 
     :param inspire_id: inspire id
     :param version: version of submission to export. If absent, returns the latest.
-    :param file_format: json, yaml, csv, root, yoda, yoda1 or original
+    :param file_format: json, yaml, csv, root, yoda, yoda1, yoda.h5 or original
     :param rivet: Rivet analysis name to override default written in YODA export
     :return: download_submission
     """
@@ -115,7 +115,7 @@ def download_submission_with_inspire_id(*args, **kwargs):
 
 @blueprint.route(f'/submission/<int:recid>/<any({FORMATS}):file_format>')
 @blueprint.route(f'/submission/<int:recid>/<int:version>/<any({FORMATS}):file_format>')
-@blueprint.route('/submission/<int:recid>/<int:version>/<any(yoda,yoda1):file_format>/<rivet>')
+@blueprint.route('/submission/<int:recid>/<int:version>/<any(yoda,yoda1,yoda.h5):file_format>/<rivet>')
 def download_submission_with_recid(*args, **kwargs):
     """
     Gets the submission file and either serves it back directly from YAML, or converts it
@@ -126,7 +126,7 @@ def download_submission_with_recid(*args, **kwargs):
 
     :param recid: submissions recid
     :param version: version of submission to export. If absent, returns the latest.
-    :param file_format: json, yaml, csv, root, yoda, yoda1 or original
+    :param file_format: json, yaml, csv, root, yoda, yoda1, yoda.h5 or original
     :param rivet: Rivet analysis name to override default written in YODA export
     :return: download_submission
     """
@@ -160,7 +160,7 @@ def download_submission(submission, file_format, offline=False, force=False, riv
     for other formats.
 
     :param submission: HEPSubmission
-    :param file_format: json, yaml, csv, root, yoda, yoda1 or original
+    :param file_format: json, yaml, csv, root, yoda, yoda1, yoda.h5 or original
     :param offline: offline creation of the conversion when a record is finalised
     :param force: force recreation of the conversion
     :param rivet_analysis_name: Rivet analysis name to override default written in YODA export
@@ -188,7 +188,8 @@ def download_submission(submission, file_format, offline=False, force=False, riv
     if file_format == 'original':
         file_format_and_extension = os.path.splitext(data_filepath)[1]
     else:
-        file_format_and_extension = '-{0}.tar.gz'.format(file_format)
+        file_format_dashed = file_format.replace('.', '-')
+        file_format_and_extension = '-{0}.tar.gz'.format(file_format_dashed)
 
     output_file = 'HEPData-{0}-v{1}{2}'.format(file_identifier, submission.version, file_format_and_extension)
 
@@ -219,10 +220,11 @@ def download_submission(submission, file_format, offline=False, force=False, riv
             print('File created at {0}'.format(output_path))
             return
 
+    file_format_dashed = file_format.replace('.', '-')
     converter_options = {
         'input_format': 'yaml',
         'output_format': file_format,
-        'filename': 'HEPData-{0}-v{1}-{2}'.format(file_identifier, submission.version, file_format),
+        'filename': 'HEPData-{0}-v{1}-{2}'.format(file_identifier, submission.version, file_format_dashed),
         'validator_schema_version': '0.1.0',
     }
 
@@ -253,7 +255,7 @@ def download_submission(submission, file_format, offline=False, force=False, riv
 
 @blueprint.route(f'/table/<inspire_id>/<path:table_name>/<any({FORMATS}):file_format>')
 @blueprint.route(f'/table/<inspire_id>/<path:table_name>/<int:version>/<any({FORMATS}):file_format>')
-@blueprint.route('/table/<inspire_id>/<path:table_name>/<int:version>/<any(yoda,yoda1):file_format>/<rivet>')
+@blueprint.route('/table/<inspire_id>/<path:table_name>/<int:version>/<any(yoda,yoda1,yoda.h5):file_format>/<rivet>')
 def download_data_table_by_inspire_id(*args, **kwargs):
     """
     Downloads the latest data file given the url ``/download/submission/ins1283842/Table 1/yaml`` or
@@ -324,7 +326,7 @@ def download_data_table_by_inspire_id(*args, **kwargs):
 
 @blueprint.route(f'/table/<int:recid>/<path:table_name>/<any({FORMATS}):file_format>')
 @blueprint.route(f'/table/<int:recid>/<path:table_name>/<int:version>/<any({FORMATS}):file_format>')
-@blueprint.route('/table/<int:recid>/<path:table_name>/<int:version>/<any(yoda,yoda1):file_format>/<rivet>')
+@blueprint.route('/table/<int:recid>/<path:table_name>/<int:version>/<any(yoda,yoda1,yoda.h5):file_format>/<rivet>')
 def download_data_table_by_recid(*args, **kwargs):
     """
     Record ID download.

hepdata/modules/records/utils/data_processing_utils.py

@@ -24,7 +24,7 @@
 
 from flask import current_app
 from collections import OrderedDict
-import re
+from hepdata.config import CFG_SUPPORTED_FORMATS
 
 from hepdata.utils.miscellaneous import sanitize_html
 
@@ -311,18 +311,10 @@ def process_ctx(ctx, light_mode=False):
 
                 _cleaned_table_name = data_table['name'].replace('%', '%25').replace('\\', '%5C')
 
-                data_table['data'] = {
-                    'json': '{0}/download/table/{1}/{2}/json'.format(
-                        site_url, _recid, _cleaned_table_name),
-                    'root': '{0}/download/table/{1}/{2}/root'.format(
-                        site_url, _recid, _cleaned_table_name),
-                    'csv': '{0}/download/table/{1}/{2}/csv'.format(
-                        site_url, _recid, _cleaned_table_name),
-                    'yoda': '{0}/download/table/{1}/{2}/yoda'.format(
-                        site_url, _recid, _cleaned_table_name),
-                    'yoda1': '{0}/download/table/{1}/{2}/yoda1'.format(
-                        site_url, _recid, _cleaned_table_name),
-                    'yaml': '{0}/download/table/{1}/{2}/yaml'.format(
-                        site_url, _recid, _cleaned_table_name)}
+                data_table['data'] = {}
+                for file_format in ['json'] + CFG_SUPPORTED_FORMATS:
+                    if file_format != 'original':
+                        data_table['data'][file_format] = '{0}/download/table/{1}/{2}/{3}'.format(
+                            site_url, _recid, _cleaned_table_name, file_format)
 
     return ctx

hepdata/modules/records/utils/json_ld.py

@@ -183,7 +183,8 @@ def _add_table_info(data, ctx, data_submission):
         'root': 'https://root.cern',
         'yaml': 'https://yaml.org',
         'csv': 'text/csv',
-        'yoda': 'https://yoda.hepforge.org'
+        'yoda': 'https://yoda.hepforge.org',
+        'yoda.h5': 'https://yoda.hepforge.org'
     }
     for download_type, format in download_types.items():
         data_downloads.append({

hepdata/modules/theme/templates/hepdata_theme/pages/formats.html

@@ -109,6 +109,7 @@ <h3>Detailed record</h3>
                     toolkit. Again, the appropriate <a href="https://yoda.hepforge.org" target="_blank" style="text-decoration: underline">YODA</a> object
                     is written according to the number of independent variables in a table. The default <code>yoda</code> output is now
                     in the YODA2 format, but there is still an option <code>yoda1</code> to output the legacy YODA1 format.
+                    Alternatively, the output can be given in the <code>yoda.h5</code> format, designed for parallel computing.
                   </li>
                 </ul>
                 <p>
@@ -123,6 +124,7 @@ <h3>Detailed record</h3>
                   <li><code>?format=root</code></li>
                   <li><code>?format=yoda</code></li>
                   <li><code>?format=yoda1</code></li>
+                  <li><code>?format=yoda.h5</code></li>
                 </ul>
                 <p>Optional parameters can also be added (separated by a <code>&</code> symbol):</p>
                 <ul>
@@ -140,7 +142,7 @@ <h3>Detailed record</h3>
                     the data tables from the response.
                   </li>
                   <li>
-                    <code>rivet=ALICE_2016_I1419244</code>: when using <code>format=yoda</code> or <code>format=yoda1</code>, specify the desired
+                    <code>rivet=ALICE_2016_I1419244</code>: when using <code>format=yoda</code>, <code>format=yoda1</code>, or <code>format=yoda.h5</code>, specify the desired
                     Rivet analysis name to be written in the YODA files if it does not match the automatically generated name.
                   </li>
                 </ul>

hepdata/version.py

@@ -28,4 +28,4 @@
 and parsed by ``setup.py``.
 """
 
-__version__ = "0.9.4dev20250813"
+__version__ = "0.9.4dev20250819"

tests/e2e/test_general.py

@@ -94,6 +94,18 @@ def test_home(app, live_server, env_browser, e2e_identifiers):
     # Close download dropdown by clicking again
     browser.find_element(By.ID, 'dLabel').click()
 
+    # Access the JSON format of the record
+    json_url = browser.find_element(By.ID, 'jsonLabel').get_attribute('href')
+    assert json_url.endswith('?format=json')
+    response = requests.get(json_url)
+    assert response.status_code == 200
+    json_data = response.json()
+    assert len(json_data['data_tables']) == 14
+    response = requests.get(json_url + '&light=true')
+    assert response.status_code == 200
+    json_data = response.json()
+    assert 'data_tables' not in json_data
+
     # Go back to homepage and click on 1st link - should be record with resources
     browser.back()
     latest_item = browser.find_elements(By.CSS_SELECTOR, '.latest-record .title')[0]
@@ -269,7 +281,8 @@ def test_accept_headers(app, live_server, e2e_identifiers):
         {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/root', 'description': 'ROOT file', 'encodingFormat': 'https://root.cern'},
         {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/yaml', 'description': 'YAML file', 'encodingFormat': 'https://yaml.org'},
         {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/csv', 'description': 'CSV file', 'encodingFormat': 'text/csv'},
-        {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/yoda', 'description': 'YODA file', 'encodingFormat': 'https://yoda.hepforge.org'}
+        {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/yoda', 'description': 'YODA file', 'encodingFormat': 'https://yoda.hepforge.org'},
+        {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/yoda.h5', 'description': 'YODA.H5 file', 'encodingFormat': 'https://yoda.hepforge.org'}
     ]
 
     # Data resource landing page (use last submission, which has resources)

tests/records_test.py

@@ -919,6 +919,12 @@ def test_get_json_ld(app, load_default_data, identifiers):
             'contentUrl': f'http://localhost:5000/download/table/2/yoda',
             'description': 'YODA file',
             'encodingFormat': 'https://yoda.hepforge.org'
+        },
+        {
+            '@type': 'DataDownload',
+            'contentUrl': f'http://localhost:5000/download/table/2/yoda.h5',
+            'description': 'YODA.H5 file',
+            'encodingFormat': 'https://yoda.hepforge.org'
         }
     ]
     assert table_data['includedInDataCatalog'] == {