diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 5fb573258..f8ba64de3 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -80,7 +80,12 @@ Ready to contribute? Here's how to set up ``hepdata`` for local development. $ git commit -s -m "Your detailed description of your changes." $ git push origin name-of-your-bugfix-or-feature -7. Submit a pull request through the GitHub website, perhaps initially as a +7. The continuous integration (CI) run via GitHub Actions requires access to secrets only available to the +https://github.com/HEPData/hepdata repository, not to forks. Therefore, pull requests should be made only +from branches of this repository, not from forks. You can request Write access to the repository by +sending an email to ``info@hepdata.net``. + +8. Submit a pull request through the GitHub website, perhaps initially as a `draft pull request `_ until you make sure that all checks have passed. diff --git a/hepdata/config.py b/hepdata/config.py index 9065b9d39..cbbc39de3 100644 --- a/hepdata/config.py +++ b/hepdata/config.py @@ -196,7 +196,7 @@ def _(x): CFG_SEARCH_RANGE_TERMS = ["recid", "publication_recid", "inspire_id"] # Possible terms used to OpenSearch API range searches CFG_CONVERTER_URL = 'https://converter.hepdata.net' -CFG_SUPPORTED_FORMATS = ['yaml', 'root', 'csv', 'yoda', 'yoda1', 'original'] +CFG_SUPPORTED_FORMATS = ['yaml', 'root', 'csv', 'yoda', 'yoda1', 'yoda.h5', 'original'] CFG_CONVERTER_TIMEOUT = 220 # timeout in seconds CFG_TMPDIR = tempfile.gettempdir() diff --git a/hepdata/ext/opensearch/document_enhancers.py b/hepdata/ext/opensearch/document_enhancers.py index 52d68fdf5..50540d9f2 100644 --- a/hepdata/ext/opensearch/document_enhancers.py +++ b/hepdata/ext/opensearch/document_enhancers.py @@ -37,7 +37,7 @@ from hepdata.modules.submission.models import DataSubmission from hepdata.utils.miscellaneous import get_resource_data -FORMATS = ['json', 'root', 'yaml', 'csv', 'yoda'] +FORMATS = ['json', 'root', 'yaml', 'csv', 'yoda', 'yoda.h5'] logging.basicConfig() log = logging.getLogger(__name__) @@ -55,7 +55,7 @@ def add_data_submission_urls(doc): doc['access_urls'] = {'links': {}} for format in FORMATS: - doc['access_urls']['links'][format] = '{0}/download/submission/ins{1}/{2}/{3}'.format( + doc['access_urls']['links'][format.replace('.', '-')] = '{0}/download/submission/ins{1}/{2}/{3}'.format( current_app.config.get('SITE_URL', 'https://www.hepdata.net'), doc['inspire_id'], doc['version'] if 'version' in doc else 1, format) @@ -67,7 +67,7 @@ def add_data_table_urls(doc): _cleaned_table_name = doc['title'].replace('%', '%25').replace('\\', '%5C') - doc['access_urls']['links'][format] = '{0}/download/table/ins{1}/{2}/{3}'.format( + doc['access_urls']['links'][format.replace('.', '-')] = '{0}/download/table/ins{1}/{2}/{3}'.format( current_app.config.get('SITE_URL', 'https://www.hepdata.net'), doc['inspire_id'], _cleaned_table_name, format) diff --git a/hepdata/modules/converter/views.py b/hepdata/modules/converter/views.py index 8b808deea..fd3c0654b 100644 --- a/hepdata/modules/converter/views.py +++ b/hepdata/modules/converter/views.py @@ -60,7 +60,7 @@ @blueprint.route(f'/submission//') @blueprint.route(f'/submission///') -@blueprint.route('/submission////') +@blueprint.route('/submission////') def download_submission_with_inspire_id(*args, **kwargs): """ Gets the submission file and either serves it back directly from YAML, or converts it @@ -71,7 +71,7 @@ def download_submission_with_inspire_id(*args, **kwargs): :param inspire_id: inspire id :param version: version of submission to export. If absent, returns the latest. - :param file_format: json, yaml, csv, root, yoda, yoda1 or original + :param file_format: json, yaml, csv, root, yoda, yoda1, yoda.h5 or original :param rivet: Rivet analysis name to override default written in YODA export :return: download_submission """ @@ -115,7 +115,7 @@ def download_submission_with_inspire_id(*args, **kwargs): @blueprint.route(f'/submission//') @blueprint.route(f'/submission///') -@blueprint.route('/submission////') +@blueprint.route('/submission////') def download_submission_with_recid(*args, **kwargs): """ Gets the submission file and either serves it back directly from YAML, or converts it @@ -126,7 +126,7 @@ def download_submission_with_recid(*args, **kwargs): :param recid: submissions recid :param version: version of submission to export. If absent, returns the latest. - :param file_format: json, yaml, csv, root, yoda, yoda1 or original + :param file_format: json, yaml, csv, root, yoda, yoda1, yoda.h5 or original :param rivet: Rivet analysis name to override default written in YODA export :return: download_submission """ @@ -160,7 +160,7 @@ def download_submission(submission, file_format, offline=False, force=False, riv for other formats. :param submission: HEPSubmission - :param file_format: json, yaml, csv, root, yoda, yoda1 or original + :param file_format: json, yaml, csv, root, yoda, yoda1, yoda.h5 or original :param offline: offline creation of the conversion when a record is finalised :param force: force recreation of the conversion :param rivet_analysis_name: Rivet analysis name to override default written in YODA export @@ -188,7 +188,8 @@ def download_submission(submission, file_format, offline=False, force=False, riv if file_format == 'original': file_format_and_extension = os.path.splitext(data_filepath)[1] else: - file_format_and_extension = '-{0}.tar.gz'.format(file_format) + file_format_dashed = file_format.replace('.', '-') + file_format_and_extension = '-{0}.tar.gz'.format(file_format_dashed) output_file = 'HEPData-{0}-v{1}{2}'.format(file_identifier, submission.version, file_format_and_extension) @@ -219,10 +220,11 @@ def download_submission(submission, file_format, offline=False, force=False, riv print('File created at {0}'.format(output_path)) return + file_format_dashed = file_format.replace('.', '-') converter_options = { 'input_format': 'yaml', 'output_format': file_format, - 'filename': 'HEPData-{0}-v{1}-{2}'.format(file_identifier, submission.version, file_format), + 'filename': 'HEPData-{0}-v{1}-{2}'.format(file_identifier, submission.version, file_format_dashed), 'validator_schema_version': '0.1.0', } @@ -253,7 +255,7 @@ def download_submission(submission, file_format, offline=False, force=False, riv @blueprint.route(f'/table///') @blueprint.route(f'/table////') -@blueprint.route('/table/////') +@blueprint.route('/table/////') def download_data_table_by_inspire_id(*args, **kwargs): """ Downloads the latest data file given the url ``/download/submission/ins1283842/Table 1/yaml`` or @@ -324,7 +326,7 @@ def download_data_table_by_inspire_id(*args, **kwargs): @blueprint.route(f'/table///') @blueprint.route(f'/table////') -@blueprint.route('/table/////') +@blueprint.route('/table/////') def download_data_table_by_recid(*args, **kwargs): """ Record ID download. diff --git a/hepdata/modules/records/utils/data_processing_utils.py b/hepdata/modules/records/utils/data_processing_utils.py index d8c492049..6d60585b0 100644 --- a/hepdata/modules/records/utils/data_processing_utils.py +++ b/hepdata/modules/records/utils/data_processing_utils.py @@ -24,7 +24,7 @@ from flask import current_app from collections import OrderedDict -import re +from hepdata.config import CFG_SUPPORTED_FORMATS from hepdata.utils.miscellaneous import sanitize_html @@ -311,18 +311,10 @@ def process_ctx(ctx, light_mode=False): _cleaned_table_name = data_table['name'].replace('%', '%25').replace('\\', '%5C') - data_table['data'] = { - 'json': '{0}/download/table/{1}/{2}/json'.format( - site_url, _recid, _cleaned_table_name), - 'root': '{0}/download/table/{1}/{2}/root'.format( - site_url, _recid, _cleaned_table_name), - 'csv': '{0}/download/table/{1}/{2}/csv'.format( - site_url, _recid, _cleaned_table_name), - 'yoda': '{0}/download/table/{1}/{2}/yoda'.format( - site_url, _recid, _cleaned_table_name), - 'yoda1': '{0}/download/table/{1}/{2}/yoda1'.format( - site_url, _recid, _cleaned_table_name), - 'yaml': '{0}/download/table/{1}/{2}/yaml'.format( - site_url, _recid, _cleaned_table_name)} + data_table['data'] = {} + for file_format in ['json'] + CFG_SUPPORTED_FORMATS: + if file_format != 'original': + data_table['data'][file_format] = '{0}/download/table/{1}/{2}/{3}'.format( + site_url, _recid, _cleaned_table_name, file_format) return ctx diff --git a/hepdata/modules/records/utils/json_ld.py b/hepdata/modules/records/utils/json_ld.py index cb4458e7c..51326f78f 100644 --- a/hepdata/modules/records/utils/json_ld.py +++ b/hepdata/modules/records/utils/json_ld.py @@ -183,7 +183,8 @@ def _add_table_info(data, ctx, data_submission): 'root': 'https://root.cern', 'yaml': 'https://yaml.org', 'csv': 'text/csv', - 'yoda': 'https://yoda.hepforge.org' + 'yoda': 'https://yoda.hepforge.org', + 'yoda.h5': 'https://yoda.hepforge.org' } for download_type, format in download_types.items(): data_downloads.append({ diff --git a/hepdata/modules/theme/templates/hepdata_theme/pages/formats.html b/hepdata/modules/theme/templates/hepdata_theme/pages/formats.html index 054d8b26b..d3fd2d3f3 100644 --- a/hepdata/modules/theme/templates/hepdata_theme/pages/formats.html +++ b/hepdata/modules/theme/templates/hepdata_theme/pages/formats.html @@ -109,6 +109,7 @@

Detailed record

toolkit. Again, the appropriate YODA object is written according to the number of independent variables in a table. The default yoda output is now in the YODA2 format, but there is still an option yoda1 to output the legacy YODA1 format. + Alternatively, the output can be given in the yoda.h5 format, designed for parallel computing.

@@ -123,6 +124,7 @@

Detailed record

  • ?format=root
  • ?format=yoda
  • ?format=yoda1
    • +
  • ?format=yoda.h5

    • Optional parameters can also be added (separated by a & symbol):

      @@ -140,7 +142,7 @@

      Detailed record

      the data tables from the response.
    • - rivet=ALICE_2016_I1419244: when using format=yoda or format=yoda1, specify the desired + rivet=ALICE_2016_I1419244: when using format=yoda, format=yoda1, or format=yoda.h5, specify the desired Rivet analysis name to be written in the YODA files if it does not match the automatically generated name.
    diff --git a/hepdata/version.py b/hepdata/version.py index 809d9ade7..2b90216b4 100644 --- a/hepdata/version.py +++ b/hepdata/version.py @@ -28,4 +28,4 @@ and parsed by ``setup.py``. """ -__version__ = "0.9.4dev20250813" +__version__ = "0.9.4dev20250819" diff --git a/tests/e2e/test_general.py b/tests/e2e/test_general.py index dfe919eb4..c31c6e738 100644 --- a/tests/e2e/test_general.py +++ b/tests/e2e/test_general.py @@ -94,6 +94,18 @@ def test_home(app, live_server, env_browser, e2e_identifiers): # Close download dropdown by clicking again browser.find_element(By.ID, 'dLabel').click() + # Access the JSON format of the record + json_url = browser.find_element(By.ID, 'jsonLabel').get_attribute('href') + assert json_url.endswith('?format=json') + response = requests.get(json_url) + assert response.status_code == 200 + json_data = response.json() + assert len(json_data['data_tables']) == 14 + response = requests.get(json_url + '&light=true') + assert response.status_code == 200 + json_data = response.json() + assert 'data_tables' not in json_data + # Go back to homepage and click on 1st link - should be record with resources browser.back() latest_item = browser.find_elements(By.CSS_SELECTOR, '.latest-record .title')[0] @@ -269,7 +281,8 @@ def test_accept_headers(app, live_server, e2e_identifiers): {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/root', 'description': 'ROOT file', 'encodingFormat': 'https://root.cern'}, {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/yaml', 'description': 'YAML file', 'encodingFormat': 'https://yaml.org'}, {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/csv', 'description': 'CSV file', 'encodingFormat': 'text/csv'}, - {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/yoda', 'description': 'YODA file', 'encodingFormat': 'https://yoda.hepforge.org'} + {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/yoda', 'description': 'YODA file', 'encodingFormat': 'https://yoda.hepforge.org'}, + {'@type': 'DataDownload', 'contentUrl': 'http://localhost:5000/download/table/1/yoda.h5', 'description': 'YODA.H5 file', 'encodingFormat': 'https://yoda.hepforge.org'} ] # Data resource landing page (use last submission, which has resources) diff --git a/tests/records_test.py b/tests/records_test.py index d701ae69f..1f6e2cc6f 100644 --- a/tests/records_test.py +++ b/tests/records_test.py @@ -919,6 +919,12 @@ def test_get_json_ld(app, load_default_data, identifiers): 'contentUrl': f'http://localhost:5000/download/table/2/yoda', 'description': 'YODA file', 'encodingFormat': 'https://yoda.hepforge.org' + }, + { + '@type': 'DataDownload', + 'contentUrl': f'http://localhost:5000/download/table/2/yoda.h5', + 'description': 'YODA.H5 file', + 'encodingFormat': 'https://yoda.hepforge.org' } ] assert table_data['includedInDataCatalog'] == {