From 272fb2305027c0cab665572d131cfaeed9dfc268 Mon Sep 17 00:00:00 2001 From: Martin Habedank Date: Thu, 26 Jun 2025 18:04:28 +0100 Subject: [PATCH 01/10] first draft to list GAMBIT analyses --- hepdata/cli.py | 4 +- hepdata/config.py | 3 + hepdata/ext/opensearch/document_enhancers.py | 2 +- .../records/assets/js/hepdata_common.js | 1 + .../components/resources-widget.html | 1 + hepdata/modules/records/utils/analyses.py | 210 +++++++++++++----- .../hepdata_search/modals/search_help.html | 7 + tests/records_test.py | 8 + 8 files changed, 173 insertions(+), 63 deletions(-) diff --git a/hepdata/cli.py b/hepdata/cli.py index b3ddbc86..32f848ec 100644 --- a/hepdata/cli.py +++ b/hepdata/cli.py @@ -223,9 +223,9 @@ def do_unload(records_to_unload): @utils.command() @with_appcontext -@click.option('--endpoint', '-e', type=str, help='Specific endpoint to update (e.g. "rivet" or "MadAnalysis" or "SModelS" or "CheckMATE" or "HackAnalysis" or "Combine"). Omit for all.') +@click.option('--endpoint', '-e', type=str, help='Specific endpoint to update (e.g. "rivet" or "MadAnalysis" or "SModelS" or "CheckMATE" or "HackAnalysis" or "Combine" or "GAMBIT"). Omit for all.') def find_and_add_record_analyses(endpoint): - """Finds analyses such as Rivet, MadAnalysis 5, SModelS, CheckMATE, HackAnalysis and Combine and adds them to records.""" + """Finds analyses such as Rivet, MadAnalysis 5, SModelS, CheckMATE, HackAnalysis, Combine and GAMBIT and adds them to records.""" update_analyses(endpoint) diff --git a/hepdata/config.py b/hepdata/config.py index 9065b9d3..90d5d42e 100644 --- a/hepdata/config.py +++ b/hepdata/config.py @@ -361,6 +361,9 @@ def _(x): 'url': 'https://creativecommons.org/licenses/by/4.0' }, }, + 'GAMBIT': { + 'endpoint_url': 'https://gambitbsm.org/analyses.json', + }, #'ufo': {}, #'xfitter': {}, #'applgrid': {}, diff --git a/hepdata/ext/opensearch/document_enhancers.py b/hepdata/ext/opensearch/document_enhancers.py index 52d68fdf..8d107765 100644 --- a/hepdata/ext/opensearch/document_enhancers.py +++ b/hepdata/ext/opensearch/document_enhancers.py @@ -94,7 +94,7 @@ def add_shortened_authors(doc): def add_analyses(doc): """ - Add analyses links such as Rivet, MadAnalysis 5, SModelS, CheckMATE, HackAnalysis, Combine, HistFactory and NUISANCE to the index. + Add analyses links such as Rivet, MadAnalysis 5, SModelS, CheckMATE, HackAnalysis, Combine, HistFactory, NUISANCE and GAMBIT to the index. :param doc: :return: diff --git a/hepdata/modules/records/assets/js/hepdata_common.js b/hepdata/modules/records/assets/js/hepdata_common.js index 04027b4b..833ecd32 100644 --- a/hepdata/modules/records/assets/js/hepdata_common.js +++ b/hepdata/modules/records/assets/js/hepdata_common.js @@ -49,6 +49,7 @@ HEPDATA.file_type_to_details = { "checkmate": {"icon": "area-chart", "description": "CheckMATE Analysis"}, "hackanalysis": {"icon": "area-chart", "description": "HackAnalysis Analysis"}, "combine": {"icon": "area-chart", "description": "Combine Analysis"}, + "gambit": {"icon": "area-chart", "description": "GAMBIT analysis"}, "xfitter": {"icon": "area-chart", "description": "xFitter Analysis"}, "applgrid": {"icon": "area-chart", "description": "APPLgrid Analysis"}, "ufo": {"icon": "rocket", "description": "Universal Feynrules Output (UFO)"}, diff --git a/hepdata/modules/records/templates/hepdata_records/components/resources-widget.html b/hepdata/modules/records/templates/hepdata_records/components/resources-widget.html index 79b35cb0..c8f79c03 100644 --- a/hepdata/modules/records/templates/hepdata_records/components/resources-widget.html +++ b/hepdata/modules/records/templates/hepdata_records/components/resources-widget.html @@ -45,6 +45,7 @@

Add Resource for Submission

+ diff --git a/hepdata/modules/records/utils/analyses.py b/hepdata/modules/records/utils/analyses.py index 188949e0..4505a8b0 100644 --- a/hepdata/modules/records/utils/analyses.py +++ b/hepdata/modules/records/utils/analyses.py @@ -44,11 +44,11 @@ @shared_task def update_analyses(endpoint=None): """ - Update (Rivet, MadAnalysis 5, SModelS, CheckMATE, HackAnalysis and Combine) analyses and remove outdated resources. + Update (Rivet, MadAnalysis 5, SModelS, CheckMATE, HackAnalysis, Combine, and GAMBIT) analyses and remove outdated resources. Allow bulk subscription to record update notifications if "subscribe_user_id" in endpoint. Add optional "description" and "license" fields if present in endpoint. - :param endpoint: either "rivet" or "MadAnalysis" or "SModelS" or "CheckMATE" or "HackAnalysis" or "Combine" or None (default) for all + :param endpoint: either "rivet" or "MadAnalysis" or "SModelS" or "CheckMATE" or "HackAnalysis" or "Combine" or "GAMBIT" or None (default) for all """ endpoints = current_app.config["ANALYSES_ENDPOINTS"] for analysis_endpoint in endpoints: @@ -64,62 +64,141 @@ def update_analyses(endpoint=None): if response and response.status_code == 200: - analyses = response.json() - analysis_resources = DataResource.query.filter_by(file_type=analysis_endpoint).all() - # Check for missing analyses. - for record in analyses: - submission = get_latest_hepsubmission(inspire_id=record, overall_status='finished') - - if submission: - num_new_resources = 0 - - for analysis in analyses[record]: - _resource_url = endpoints[analysis_endpoint]["url_template"].format(analysis) - - if not is_resource_added_to_submission(submission.publication_recid, submission.version, - _resource_url): - - log.info('Adding {} analysis to ins{} with URL {}'.format( - analysis_endpoint, record, _resource_url) - ) - new_resource = DataResource( - file_location=_resource_url, - file_type=analysis_endpoint) - - if "description" in endpoints[analysis_endpoint]: - new_resource.file_description = str(endpoints[analysis_endpoint]["description"]) - - if "license" in endpoints[analysis_endpoint]: - resource_license = get_license(endpoints[analysis_endpoint]["license"]) - new_resource.file_license = resource_license.id - - submission.resources.append(new_resource) - num_new_resources += 1 - - else: - - # Remove resources from 'analysis_resources' list. - resources = list(filter(lambda a: a.file_location == _resource_url, analysis_resources)) - for resource in resources: - analysis_resources.remove(resource) - - if num_new_resources: - - try: - db.session.add(submission) - db.session.commit() - latest_submission = get_latest_hepsubmission(inspire_id=record) - if submission.version == latest_submission.version: - index_record_ids([submission.publication_recid]) - except Exception as e: - db.session.rollback() - log.error(e) - - else: - log.debug("An analysis is available in {0} but with no equivalent in HEPData (ins{1}).".format( - analysis_endpoint, record)) + r_json = response.json() + + if "analyses" in r_json: # new JSON file + print("r_json", r_json) + + # Check for missing analyses. + for ana in r_json["analyses"]: + print("ana", ana) + inspire_id = ana["inspire_id"] + submission = get_latest_hepsubmission(inspire_id=str(inspire_id), overall_status='finished') # TODO: make inspire_id an int + + if submission: + print("submission", submission) + num_new_resources = 0 + + print("ana['implementations']", [ana["implementations"]]) + + ana["implementations"] = [ana["implementations"]] # hack to get GAMBIT to follow standard + + for implementation in ana["implementations"]: + # 'url_templates': {'main_url': 'https://github.com/GambitBSM/gambit/blob/master/ColliderBit/src/analyses/Analysis_{name}.cpp' + print(implementation) + ana_name = implementation["name"] + ana_path = implementation["path"] if "path" in implementation else "" + _resource_url = r_json["url_templates"]["main_url"] + _resource_url = _resource_url.replace("gambit", "gambit_2.6") + prev_url = None + n_tries, max_tries = 0, 10 + while _resource_url!=prev_url and n_triesOther useful searches (ProSelecta analysis for use with NUISANCE) +
  • + analysis:GAMBIT + + (GAMBIT analysis) + +
    • diff --git a/tests/records_test.py b/tests/records_test.py index d701ae69..181b7324 100644 --- a/tests/records_test.py +++ b/tests/records_test.py @@ -1117,6 +1117,14 @@ def test_update_analyses(app): assert license_data.name == 'cc-by-4.0' assert license_data.url == 'https://creativecommons.org/licenses/by/4.0' + # Import a record that has an associated GAMBIT analysis + import_records(['ins1827025'], synchronous=True) + analysis_resources = DataResource.query.filter_by(file_type='GAMBIT').all() + assert len(analysis_resources) == 0 + update_analyses('GAMBIT') + analysis_resources = DataResource.query.filter_by(file_type='GAMBIT').all() + assert len(analysis_resources) == 1 + assert analysis_resources[0].file_location == 'https://github.com/GambitBSM/gambit_2.6/blob/release_2.6/ColliderBit/src/analyses/Analysis_ATLAS_13TeV_0LEP_139invfb.cpp' def test_generate_license_data_by_id(app): """ From 528e328a370857e180a039f1ee92c94649339a8f Mon Sep 17 00:00:00 2001 From: Martin Habedank Date: Mon, 30 Jun 2025 10:12:38 +0100 Subject: [PATCH 02/10] remove GAMBIT hacks and debug printouts --- hepdata/modules/records/utils/analyses.py | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/hepdata/modules/records/utils/analyses.py b/hepdata/modules/records/utils/analyses.py index 4505a8b0..2217a303 100644 --- a/hepdata/modules/records/utils/analyses.py +++ b/hepdata/modules/records/utils/analyses.py @@ -67,38 +67,28 @@ def update_analyses(endpoint=None): analysis_resources = DataResource.query.filter_by(file_type=analysis_endpoint).all() r_json = response.json() + new_json = "analyses" in r_json - if "analyses" in r_json: # new JSON file - print("r_json", r_json) + if new_json: # Check for missing analyses. for ana in r_json["analyses"]: - print("ana", ana) inspire_id = ana["inspire_id"] submission = get_latest_hepsubmission(inspire_id=str(inspire_id), overall_status='finished') # TODO: make inspire_id an int if submission: - print("submission", submission) num_new_resources = 0 - print("ana['implementations']", [ana["implementations"]]) - - ana["implementations"] = [ana["implementations"]] # hack to get GAMBIT to follow standard - for implementation in ana["implementations"]: - # 'url_templates': {'main_url': 'https://github.com/GambitBSM/gambit/blob/master/ColliderBit/src/analyses/Analysis_{name}.cpp' - print(implementation) ana_name = implementation["name"] ana_path = implementation["path"] if "path" in implementation else "" _resource_url = r_json["url_templates"]["main_url"] - _resource_url = _resource_url.replace("gambit", "gambit_2.6") prev_url = None n_tries, max_tries = 0, 10 while _resource_url!=prev_url and n_tries Date: Tue, 9 Sep 2025 15:39:12 +0100 Subject: [PATCH 03/10] shorten tool lists --- hepdata/cli.py | 8 ++++---- hepdata/ext/opensearch/document_enhancers.py | 2 +- hepdata/modules/records/utils/analyses.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/hepdata/cli.py b/hepdata/cli.py index 32f848ec..df6aa806 100644 --- a/hepdata/cli.py +++ b/hepdata/cli.py @@ -35,7 +35,7 @@ from hepdata.modules.submission.models import HEPSubmission from hepdata.modules.submission.api import get_latest_hepsubmission from .factory import create_app -from hepdata.config import CFG_PUB_TYPE +from hepdata.config import CFG_PUB_TYPE, ANALYSES_ENDPOINTS from hepdata.ext.opensearch.api import reindex_all, get_records_matching_field from hepdata.modules.records.importer import api as importer_api from hepdata.modules.records.utils import data_files @@ -55,7 +55,7 @@ cli = create_cli(create_app=create_app) default_recids = 'ins1283842,ins1245023,ins1311487' - +endpoints_as_string = '"' + '" or "'.join(ANALYSES_ENDPOINTS.keys()) + '"' @cli.group() def importer(): @@ -223,9 +223,9 @@ def do_unload(records_to_unload): @utils.command() @with_appcontext -@click.option('--endpoint', '-e', type=str, help='Specific endpoint to update (e.g. "rivet" or "MadAnalysis" or "SModelS" or "CheckMATE" or "HackAnalysis" or "Combine" or "GAMBIT"). Omit for all.') +@click.option('--endpoint', '-e', type=str, help=f'Specific endpoint to update (e.g. {endpoints_as_string}). Omit for all.') def find_and_add_record_analyses(endpoint): - """Finds analyses such as Rivet, MadAnalysis 5, SModelS, CheckMATE, HackAnalysis, Combine and GAMBIT and adds them to records.""" + """Finds analyses from tools such as Rivet, MadAnalysis 5, etc. and adds them to records.""" update_analyses(endpoint) diff --git a/hepdata/ext/opensearch/document_enhancers.py b/hepdata/ext/opensearch/document_enhancers.py index 9b6c0c52..fbba8ae9 100644 --- a/hepdata/ext/opensearch/document_enhancers.py +++ b/hepdata/ext/opensearch/document_enhancers.py @@ -94,7 +94,7 @@ def add_shortened_authors(doc): def add_analyses(doc): """ - Add analyses links such as Rivet, MadAnalysis 5, SModelS, CheckMATE, HackAnalysis, Combine, HistFactory, NUISANCE and GAMBIT to the index. + Add analyses links to tools such as Rivet, MadAnalysis 5, etc. to the index. :param doc: :return: diff --git a/hepdata/modules/records/utils/analyses.py b/hepdata/modules/records/utils/analyses.py index f4d20b07..e971442f 100644 --- a/hepdata/modules/records/utils/analyses.py +++ b/hepdata/modules/records/utils/analyses.py @@ -52,11 +52,11 @@ def test_analyses_schema(json_file, schema_version="1.0.0"): @shared_task def update_analyses(endpoint=None): """ - Update (Rivet, MadAnalysis 5, SModelS, CheckMATE, HackAnalysis, Combine, and GAMBIT) analyses and remove outdated resources. + Update tools (Rivet, MadAnalysis 5, etc.) analyses and remove outdated resources. Allow bulk subscription to record update notifications if "subscribe_user_id" in endpoint. Add optional "description" and "license" fields if present in endpoint. - :param endpoint: either "rivet" or "MadAnalysis" or "SModelS" or "CheckMATE" or "HackAnalysis" or "Combine" or "GAMBIT" or None (default) for all + :param endpoint: any one from config.ANALYSES_ENDPOINTS ("rivet", "MadAnalysis", etc.) or None (default) for all """ endpoints = current_app.config["ANALYSES_ENDPOINTS"] From 23b572fed7102ea36797cddce3aa21ab4a4701d5 Mon Sep 17 00:00:00 2001 From: Martin Habedank Date: Tue, 9 Sep 2025 15:39:36 +0100 Subject: [PATCH 04/10] remove unnecessary import because there is already an analysis available --- tests/records_test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/records_test.py b/tests/records_test.py index 6280cdfc..b0b68118 100644 --- a/tests/records_test.py +++ b/tests/records_test.py @@ -1131,14 +1131,13 @@ def test_update_analyses(app): assert license_data.name == 'cc-by-4.0' assert license_data.url == 'https://creativecommons.org/licenses/by/4.0' - # Import a record that has an associated GAMBIT analysis - import_records(['ins1827025'], synchronous=True) + # ins1847779 also has a GAMBIT analysis, so don't need to import another record analysis_resources = DataResource.query.filter_by(file_type='GAMBIT').all() assert len(analysis_resources) == 0 update_analyses('GAMBIT') analysis_resources = DataResource.query.filter_by(file_type='GAMBIT').all() assert len(analysis_resources) == 1 - assert analysis_resources[0].file_location == 'https://github.com/GambitBSM/gambit_2.6/blob/release_2.6/ColliderBit/src/analyses/Analysis_ATLAS_13TeV_0LEP_139invfb.cpp' + assert analysis_resources[0].file_location == 'https://github.com/GambitBSM/gambit_2.6/blob/release_2.6/ColliderBit/src/analyses/Analysis_ATLAS_13TeV_MONOJET_139infb.cpp' # Call update_analysis using an endpoint with no endpoint_url current_app.config["ANALYSES_ENDPOINTS"]["TestAnalysis"] = {} From a4f3026a48f3e5cf3207cd1feda81b07aae5dcd2 Mon Sep 17 00:00:00 2001 From: Martin Habedank Date: Thu, 11 Sep 2025 14:45:47 +0100 Subject: [PATCH 05/10] remove link options for frameworks in config.ANALYSES_ENDPOINTS --- .../hepdata_records/components/resources-widget.html | 7 ------- 1 file changed, 7 deletions(-) diff --git a/hepdata/modules/records/templates/hepdata_records/components/resources-widget.html b/hepdata/modules/records/templates/hepdata_records/components/resources-widget.html index c8f79c03..3f83b828 100644 --- a/hepdata/modules/records/templates/hepdata_records/components/resources-widget.html +++ b/hepdata/modules/records/templates/hepdata_records/components/resources-widget.html @@ -39,13 +39,6 @@

    Add Resource for Submission