diff --git a/.github/workflows/redo_regressions.yml b/.github/workflows/redo_regressions.yml new file mode 100644 index 0000000000..c7111631aa --- /dev/null +++ b/.github/workflows/redo_regressions.yml @@ -0,0 +1,64 @@ +name: Redo regressions + +# start job only for PRs when a label is added. +on: + pull_request: + types: [labeled] + +# some general variables +env: + # https://keras.io/getting_started/faq/#how-can-i-obtain-reproducible-results-using-keras-during-development + PYTHONHASHSEED: "0" + +jobs: + build: + if: contains(github.event.pull_request.labels.*.name, 'redo-regressions') + strategy: + matrix: + os: [ubuntu-latest] + python-version: ["3.11"] + include: + - os: ubuntu-latest + CONDA_OS: linux-64 + fail-fast: false + runs-on: ${{ matrix.os }} + env: + NETRC_FILE: ${{ secrets.NETRC_FILE }} + NNPDF_SSH_KEY: ${{ secrets.NNPDF_SSH_KEY }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.ref }} + + - uses: conda-incubator/setup-miniconda@v3 + with: + python-version: ${{ matrix.python-version }} + use-mamba: true + auto-update-conda: true + activate-environment: test + - name: Setup conda and install LHAPDF and pandoc + shell: bash -l {0} + run: | + echo "$NETRC_FILE" | base64 --decode > ~/.netrc + conda config --remove channels defaults + conda config --append channels conda-forge + conda config --prepend channels https://packages.nnpdf.science/public + conda config --set show_channel_urls true + conda install lhapdf pandoc + - name: Install nnpdf with testing and qed extras + shell: bash -l {0} + run: | + pip install .[qed,tests] + - name: Regenerate regressions + shell: bash -l {0} + run: | + pytest extra_tests/regression_checks.py --regenerate True + - name: Commit + shell: bash -l {0} + run: | + git config user.name "Redo regressions bot" + git add extra_tests/regression_fits/* + git status + git commit -m "Automatically regenerated regressions from PR ${{ github.event.number }}, branch ${{ github.event.pull_request.head.ref }}." + git push origin "${{ github.event.pull_request.head.ref }}" diff --git a/extra_tests/conftest.py b/extra_tests/conftest.py new file mode 100644 index 0000000000..78ca171de3 --- /dev/null +++ b/extra_tests/conftest.py @@ -0,0 +1,14 @@ +""" +This enables the use of a command line argument to set regenerate, which is used to autmate the +regeneration of the regression data. +""" + + +def pytest_addoption(parser): + parser.addoption("--regenerate", action="store", default=False) + + +def pytest_generate_tests(metafunc): + option_value = metafunc.config.option.regenerate + if 'regenerate' in metafunc.fixturenames and option_value is not None: + metafunc.parametrize("regenerate", [option_value]) diff --git a/extra_tests/readme.md b/extra_tests/readme.md index 2d748316aa..8e912bca48 100644 --- a/extra_tests/readme.md +++ b/extra_tests/readme.md @@ -10,7 +10,7 @@ In that case the following recipe shall be applied: 2. If the changes look due to numerics, run a full production-like fit (4.0 baseline, 100 replicas, etc) 3. Review and finish the PR normally, and then, before merge: a. Rebase on top of master - b. Perform one commit with the update to the regression tests + b. Add the 'redo-regressions' label to the PR to automatically regenerate the regressions If instead, the changes are supposed to change the numerical values of the result (e.g., a change in the treatment of seeds) please document it in the release notes for the following tag. diff --git a/extra_tests/regression_checks.py b/extra_tests/regression_checks.py index 7d4dfcb817..de8da2cabe 100644 --- a/extra_tests/regression_checks.py +++ b/extra_tests/regression_checks.py @@ -2,14 +2,14 @@ Regression tests """ -from n3fit.tests.test_fit import check_fit_results, EXE import pathlib -from reportengine.compat import yaml import shutil import subprocess as sp import pytest +from n3fit.tests.test_fit import EXE, check_fit_results +from reportengine.compat import yaml REGRESSION_FOLDER = pathlib.Path(__file__).with_name("regression_fits") @@ -29,7 +29,7 @@ @pytest.mark.parametrize("runcard,replica", runcard_and_replicas.items()) -def test_regression_fit(tmp_path, runcard, replica): +def test_regression_fit(tmp_path, runcard, replica, regenerate): runcard_name = f"{runcard}.yml" runcard_file = REGRESSION_FOLDER / runcard_name shutil.copy(runcard_file, tmp_path) @@ -41,4 +41,6 @@ def test_regression_fit(tmp_path, runcard, replica): sp.run(f"{EXE} {runcard_name} {replica}".split(), cwd=tmp_path, check=True) old_json_file = REGRESSION_FOLDER / f"{runcard}_{replica}.json" - check_fit_results(tmp_path, runcard, replica, old_json_file, regenerate=False, rel_error=1e-2) + check_fit_results( + tmp_path, runcard, replica, old_json_file, regenerate=regenerate, rel_error=1e-2 + ) diff --git a/extra_tests/regression_fits/central_16.json b/extra_tests/regression_fits/central_16.json index 0376b99305..8954787d8c 100644 --- a/extra_tests/regression_fits/central_16.json +++ b/extra_tests/regression_fits/central_16.json @@ -63,32 +63,32 @@ 1.1961095151498038 ], "integrability": [ - 0.0009203124718642486, + 0.0009203124718646927, 0.0009203124718629718, 4.8332231357695044e-05, - 0.4625758156180382, - 0.006963429623282114 + 0.4625758156180373, + 0.0069634296232816695 ], "timing": { "walltime": { - "Total": 30.79029417037964, + "Total": 18.60609221458435, "start": 0.0, - "replica_set": 0.430037260055542, - "replica_fitted": 30.790075302124023, - "replica_set_to_replica_fitted": 30.36003804206848 + "replica_set": 0.3607151508331299, + "replica_fitted": 18.606019735336304, + "replica_set_to_replica_fitted": 18.245304584503174 }, "cputime": { - "Total": 58.881718801999995, + "Total": 20.6525949, "start": 0.0, - "replica_set": 5.264737561999997, - "replica_fitted": 58.881475626, - "replica_set_to_replica_fitted": 53.616738064 + "replica_set": 0.6791323, + "replica_fitted": 20.6525204, + "replica_set_to_replica_fitted": 19.9733881 } }, "version": { "tensorflow": "2.15.0, mkl=??", "numpy": "1.24.4", - "nnpdf": "4.0.8.post113.dev0+e1b5f2cd5.dirty", - "validphys": "4.0.8.post113.dev0+e1b5f2cd5.dirty" + "nnpdf": "4.0.8.post168.dev0+5adb0639b", + "validphys": "4.0.8.post168.dev0+5adb0639b" } } \ No newline at end of file diff --git a/extra_tests/regression_fits/diagonal_45.json b/extra_tests/regression_fits/diagonal_45.json index a3acbfeed3..8b87ceb346 100644 --- a/extra_tests/regression_fits/diagonal_45.json +++ b/extra_tests/regression_fits/diagonal_45.json @@ -58,37 +58,37 @@ "arc_lengths": [ 1.3182267178222185, 1.54441465374414, - 1.179107351893619, + 1.1791073518936193, 1.0015316488813237, 1.2179106362142496 ], "integrability": [ - 0.0008988642075570819, + 0.000898864207557748, 0.0008988642075570263, 4.724649568210726e-05, - 0.4626152291893959, - 0.0070505898911492615 + 0.462615229189395, + 0.007050589891148817 ], "timing": { "walltime": { - "Total": 28.264042139053345, + "Total": 17.62032151222229, "start": 0.0, - "replica_set": 0.4277496337890625, - "replica_fitted": 28.26386070251465, - "replica_set_to_replica_fitted": 27.836111068725586 + "replica_set": 0.3764665126800537, + "replica_fitted": 17.620244026184082, + "replica_set_to_replica_fitted": 17.24377751350403 }, "cputime": { - "Total": 56.531395771, + "Total": 19.6138915, "start": 0.0, - "replica_set": 5.303023015999997, - "replica_fitted": 56.53119243, - "replica_set_to_replica_fitted": 51.228169414 + "replica_set": 0.6932296000000004, + "replica_fitted": 19.6138119, + "replica_set_to_replica_fitted": 18.9205823 } }, "version": { "tensorflow": "2.15.0, mkl=??", "numpy": "1.24.4", - "nnpdf": "4.0.8.post113.dev0+e1b5f2cd5.dirty", - "validphys": "4.0.8.post113.dev0+e1b5f2cd5.dirty" + "nnpdf": "4.0.8.post168.dev0+5adb0639b", + "validphys": "4.0.8.post168.dev0+5adb0639b" } } \ No newline at end of file diff --git a/extra_tests/regression_fits/feature_scaling_81.json b/extra_tests/regression_fits/feature_scaling_81.json index 75faa050c4..fa77d5a984 100644 --- a/extra_tests/regression_fits/feature_scaling_81.json +++ b/extra_tests/regression_fits/feature_scaling_81.json @@ -56,14 +56,14 @@ "chi2": 1.4108951091766357, "pos_state": "POS_VETO", "arc_lengths": [ - 1.9705579690885775, + 1.9705579690885773, 1.512324164715868, 1.2608150353204404, 1.1147020004036432, 2.925581113420889 ], "integrability": [ - 0.0038688098720734143, + 0.0038688098720738584, 0.003868809872073914, 2.2851325695821822e-05, 0.05472333077341307, @@ -71,24 +71,24 @@ ], "timing": { "walltime": { - "Total": 28.482715845108032, + "Total": 17.96898365020752, "start": 0.0, - "replica_set": 0.42058467864990234, - "replica_fitted": 28.482331037521362, - "replica_set_to_replica_fitted": 28.06174635887146 + "replica_set": 0.3639094829559326, + "replica_fitted": 17.968825101852417, + "replica_set_to_replica_fitted": 17.604915618896484 }, "cputime": { - "Total": 53.414345739000005, + "Total": 19.9091685, "start": 0.0, - "replica_set": 5.313881866999999, - "replica_fitted": 53.413937684, - "replica_set_to_replica_fitted": 48.100055817 + "replica_set": 0.6803557000000007, + "replica_fitted": 19.9090083, + "replica_set_to_replica_fitted": 19.2286526 } }, "version": { "tensorflow": "2.15.0, mkl=??", "numpy": "1.24.4", - "nnpdf": "4.0.8.post113.dev0+e1b5f2cd5.dirty", - "validphys": "4.0.8.post113.dev0+e1b5f2cd5.dirty" + "nnpdf": "4.0.8.post168.dev0+5adb0639b", + "validphys": "4.0.8.post168.dev0+5adb0639b" } } \ No newline at end of file diff --git a/extra_tests/regression_fits/flavour_29.json b/extra_tests/regression_fits/flavour_29.json index 0468433155..bc5cc8c99d 100644 --- a/extra_tests/regression_fits/flavour_29.json +++ b/extra_tests/regression_fits/flavour_29.json @@ -63,32 +63,32 @@ 0.9907081134354656 ], "integrability": [ - 0.46764282882213526, + 0.4676428288221359, 0.46764282882213587, 0.08090896345675005, - 0.1003280878067017, + 0.10032808780670167, 0.5176757127046587 ], "timing": { "walltime": { - "Total": 9.50145959854126, + "Total": 7.240033864974976, "start": 0.0, - "replica_set": 0.4137749671936035, - "replica_fitted": 9.501240968704224, - "replica_set_to_replica_fitted": 9.08746600151062 + "replica_set": 0.37552475929260254, + "replica_fitted": 7.239804267883301, + "replica_set_to_replica_fitted": 6.864279508590698 }, "cputime": { - "Total": 17.63484543, + "Total": 7.8813866, "start": 0.0, - "replica_set": 5.07142004, - "replica_fitted": 17.634602875, - "replica_set_to_replica_fitted": 12.563182835 + "replica_set": 0.6917157000000005, + "replica_fitted": 7.8811553000000005, + "replica_set_to_replica_fitted": 7.1894396 } }, "version": { "tensorflow": "2.15.0, mkl=??", "numpy": "1.24.4", - "nnpdf": "4.0.8.post113.dev0+e1b5f2cd5.dirty", - "validphys": "4.0.8.post113.dev0+e1b5f2cd5.dirty" + "nnpdf": "4.0.8.post168.dev0+5adb0639b", + "validphys": "4.0.8.post168.dev0+5adb0639b" } } \ No newline at end of file diff --git a/extra_tests/regression_fits/no_lagrange_27.json b/extra_tests/regression_fits/no_lagrange_27.json index f4b36c47d9..aae13af206 100644 --- a/extra_tests/regression_fits/no_lagrange_27.json +++ b/extra_tests/regression_fits/no_lagrange_27.json @@ -58,37 +58,37 @@ "arc_lengths": [ 1.3221023191915289, 1.4870345587210063, - 1.1735323019190387, + 1.1735323019190385, 1.0005950030908517, 1.221984399084671 ], "integrability": [ - 0.0009426986289349792, + 0.0009426986289356454, 0.0009426986289332584, 5.00080153632565e-05, - 0.47462750226259187, - 0.007130859827156 + 0.474627502262591, + 0.007130859827155556 ], "timing": { "walltime": { - "Total": 33.483086824417114, + "Total": 21.26565957069397, "start": 0.0, - "replica_set": 0.40949511528015137, - "replica_fitted": 33.48285508155823, - "replica_set_to_replica_fitted": 33.073359966278076 + "replica_set": 0.365405797958374, + "replica_fitted": 21.265546083450317, + "replica_set_to_replica_fitted": 20.900140285491943 }, "cputime": { - "Total": 57.29150730599999, + "Total": 23.1746251, "start": 0.0, - "replica_set": 5.309074099, - "replica_fitted": 57.291249822999994, - "replica_set_to_replica_fitted": 51.982175723999994 + "replica_set": 0.6833834000000003, + "replica_fitted": 23.1745104, + "replica_set_to_replica_fitted": 22.491127 } }, "version": { "tensorflow": "2.15.0, mkl=??", "numpy": "1.24.4", - "nnpdf": "4.0.8.post113.dev0+e1b5f2cd5.dirty", - "validphys": "4.0.8.post113.dev0+e1b5f2cd5.dirty" + "nnpdf": "4.0.8.post168.dev0+5adb0639b", + "validphys": "4.0.8.post168.dev0+5adb0639b" } } \ No newline at end of file diff --git a/extra_tests/regression_fits/no_msr_92.json b/extra_tests/regression_fits/no_msr_92.json index 188d277639..fc1f4dffcf 100644 --- a/extra_tests/regression_fits/no_msr_92.json +++ b/extra_tests/regression_fits/no_msr_92.json @@ -63,32 +63,32 @@ 1.1751094831216375 ], "integrability": [ - 0.0009919884760163575, + 0.0009919884760162465, 0.000991988476016628, 4.665318033393595e-05, - 0.4762776866555214, - 0.007486370101105172 + 0.4762776866555213, + 0.007486370101105047 ], "timing": { "walltime": { - "Total": 47.275179624557495, + "Total": 29.779701471328735, "start": 0.0, - "replica_set": 0.4134683609008789, - "replica_fitted": 47.27485942840576, - "replica_set_to_replica_fitted": 46.86139106750488 + "replica_set": 0.37215209007263184, + "replica_fitted": 29.779524326324463, + "replica_set_to_replica_fitted": 29.40737223625183 }, "cputime": { - "Total": 75.49572824999998, + "Total": 31.8365965, "start": 0.0, - "replica_set": 5.314293298999999, - "replica_fitted": 75.49538364399999, - "replica_set_to_replica_fitted": 70.181090345 + "replica_set": 0.6885683, + "replica_fitted": 31.8364179, + "replica_set_to_replica_fitted": 31.1478496 } }, "version": { "tensorflow": "2.15.0, mkl=??", "numpy": "1.24.4", - "nnpdf": "4.0.8.post113.dev0+e1b5f2cd5.dirty", - "validphys": "4.0.8.post113.dev0+e1b5f2cd5.dirty" + "nnpdf": "4.0.8.post168.dev0+5adb0639b", + "validphys": "4.0.8.post168.dev0+5adb0639b" } } \ No newline at end of file diff --git a/extra_tests/regression_fits/no_sumrules_18.json b/extra_tests/regression_fits/no_sumrules_18.json index 8934996ae9..0c041c1598 100644 --- a/extra_tests/regression_fits/no_sumrules_18.json +++ b/extra_tests/regression_fits/no_sumrules_18.json @@ -67,28 +67,28 @@ 0.00022732430079495936, 1.0915414520598632e-05, 0.4772165492177008, - 0.0018179940525438198 + 0.001817994052543806 ], "timing": { "walltime": { - "Total": 37.01521110534668, + "Total": 25.12443494796753, "start": 0.0, - "replica_set": 0.4171748161315918, - "replica_fitted": 37.014857053756714, - "replica_set_to_replica_fitted": 36.59768223762512 + "replica_set": 0.3976583480834961, + "replica_fitted": 25.12428593635559, + "replica_set_to_replica_fitted": 24.726627588272095 }, "cputime": { - "Total": 60.474931141, + "Total": 27.074280200000004, "start": 0.0, - "replica_set": 5.3048759919999995, - "replica_fitted": 60.47455261100001, - "replica_set_to_replica_fitted": 55.16967661900001 + "replica_set": 0.7145655, + "replica_fitted": 27.074129300000003, + "replica_set_to_replica_fitted": 26.359563800000004 } }, "version": { "tensorflow": "2.15.0, mkl=??", "numpy": "1.24.4", - "nnpdf": "4.0.8.post113.dev0+e1b5f2cd5.dirty", - "validphys": "4.0.8.post113.dev0+e1b5f2cd5.dirty" + "nnpdf": "4.0.8.post168.dev0+5adb0639b", + "validphys": "4.0.8.post168.dev0+5adb0639b" } } \ No newline at end of file diff --git a/extra_tests/regression_fits/no_vsr_54.json b/extra_tests/regression_fits/no_vsr_54.json index 4fbd7f8f5a..beafc32837 100644 --- a/extra_tests/regression_fits/no_vsr_54.json +++ b/extra_tests/regression_fits/no_vsr_54.json @@ -56,39 +56,39 @@ "chi2": 2.158350944519043, "pos_state": "POS_VETO", "arc_lengths": [ - 1.3149937623522563, + 1.314993762352256, 1.4737261989061683, 1.0007785798672333, 0.9905270817090487, 1.0083087444694132 ], "integrability": [ - 0.00021743857451045345, + 0.00021743857451178572, 0.00021743857450995385, 1.091591110413681e-05, - 0.4652469605207441, - 0.0017131903587130903 + 0.46524696052074344, + 0.0017131903587124242 ], "timing": { "walltime": { - "Total": 30.30263924598694, + "Total": 18.69965434074402, "start": 0.0, - "replica_set": 0.4247448444366455, - "replica_fitted": 30.30237579345703, - "replica_set_to_replica_fitted": 29.877630949020386 + "replica_set": 0.388653039932251, + "replica_fitted": 18.699567556381226, + "replica_set_to_replica_fitted": 18.310914516448975 }, "cputime": { - "Total": 58.57155645499999, + "Total": 20.752095100000002, "start": 0.0, - "replica_set": 5.233104689999998, - "replica_fitted": 58.571267613, - "replica_set_to_replica_fitted": 53.338162923000006 + "replica_set": 0.7052243000000011, + "replica_fitted": 20.7520065, + "replica_set_to_replica_fitted": 20.0467822 } }, "version": { "tensorflow": "2.15.0, mkl=??", "numpy": "1.24.4", - "nnpdf": "4.0.8.post113.dev0+e1b5f2cd5.dirty", - "validphys": "4.0.8.post113.dev0+e1b5f2cd5.dirty" + "nnpdf": "4.0.8.post168.dev0+5adb0639b", + "validphys": "4.0.8.post168.dev0+5adb0639b" } } \ No newline at end of file diff --git a/extra_tests/regression_fits/normal_fit_72.json b/extra_tests/regression_fits/normal_fit_72.json index bf31a4714f..f02f237d23 100644 --- a/extra_tests/regression_fits/normal_fit_72.json +++ b/extra_tests/regression_fits/normal_fit_72.json @@ -56,39 +56,39 @@ "chi2": 2.4093308448791504, "pos_state": "POS_PASS", "arc_lengths": [ - 1.3077516647379446, + 1.3077516647379444, 1.2693092406788389, 1.1686582018448737, 1.0014198240309733, 1.1888768971089245 ], "integrability": [ - 0.0009303889601149917, + 0.0009303889601154358, 0.0009303889601145476, 4.6177159219396025e-05, - 0.48856043070554733, - 0.00679302011849281 + 0.4885604307055469, + 0.0067930201184924766 ], "timing": { "walltime": { - "Total": 41.881762981414795, + "Total": 24.668087005615234, "start": 0.0, - "replica_set": 0.4757258892059326, - "replica_fitted": 41.881460189819336, - "replica_set_to_replica_fitted": 41.4057343006134 + "replica_set": 0.37459492683410645, + "replica_fitted": 24.667866468429565, + "replica_set_to_replica_fitted": 24.29327154159546 }, "cputime": { - "Total": 78.09539571100001, + "Total": 27.1846608, "start": 0.0, - "replica_set": 5.260408395999995, - "replica_fitted": 78.09507449899999, - "replica_set_to_replica_fitted": 72.834666103 + "replica_set": 0.688195799999999, + "replica_fitted": 27.1844376, + "replica_set_to_replica_fitted": 26.4962418 } }, "version": { "tensorflow": "2.15.0, mkl=??", "numpy": "1.24.4", - "nnpdf": "4.0.8.post113.dev0+e1b5f2cd5.dirty", - "validphys": "4.0.8.post113.dev0+e1b5f2cd5.dirty" + "nnpdf": "4.0.8.post168.dev0+5adb0639b", + "validphys": "4.0.8.post168.dev0+5adb0639b" } } \ No newline at end of file diff --git a/extra_tests/regression_fits/trainable_prepro_61.json b/extra_tests/regression_fits/trainable_prepro_61.json index 4ee7ea2b43..16192808ac 100644 --- a/extra_tests/regression_fits/trainable_prepro_61.json +++ b/extra_tests/regression_fits/trainable_prepro_61.json @@ -58,37 +58,37 @@ "arc_lengths": [ 1.0440971744004135, 1.528025761219303, - 1.0033084238004522, + 1.003308423800452, 0.9979762394415533, 1.419281059365766 ], "integrability": [ - 0.11098538525402649, + 0.1109853852540276, 0.11098538525402574, 0.0002027637756321643, - 0.008434453862719238, - 0.0018140712927556812 + 0.008434453862718683, + 0.0018140712927553482 ], "timing": { "walltime": { - "Total": 31.576780557632446, + "Total": 19.53623390197754, "start": 0.0, - "replica_set": 0.41596508026123047, - "replica_fitted": 31.576586723327637, - "replica_set_to_replica_fitted": 31.160621643066406 + "replica_set": 0.4048268795013428, + "replica_fitted": 19.536094903945923, + "replica_set_to_replica_fitted": 19.13126802444458 }, "cputime": { - "Total": 59.362012572, + "Total": 21.8017812, "start": 0.0, - "replica_set": 5.353041050999998, - "replica_fitted": 59.361791798, - "replica_set_to_replica_fitted": 54.008750747 + "replica_set": 0.7218703000000009, + "replica_fitted": 21.8016403, + "replica_set_to_replica_fitted": 21.079769999999996 } }, "version": { "tensorflow": "2.15.0, mkl=??", "numpy": "1.24.4", - "nnpdf": "4.0.8.post113.dev0+e1b5f2cd5.dirty", - "validphys": "4.0.8.post113.dev0+e1b5f2cd5.dirty" + "nnpdf": "4.0.8.post168.dev0+5adb0639b", + "validphys": "4.0.8.post168.dev0+5adb0639b" } } \ No newline at end of file diff --git a/n3fit/src/n3fit/tests/test_fit.py b/n3fit/src/n3fit/tests/test_fit.py index 1427bf17e8..1ec36aa76c 100644 --- a/n3fit/src/n3fit/tests/test_fit.py +++ b/n3fit/src/n3fit/tests/test_fit.py @@ -86,7 +86,6 @@ def check_fit_results( if regenerate: shutil.copy2(new_expgrid_file, old_expgrid_file) shutil.copy2(new_json_file, regression_json) - raise FileNotFoundError("Regression test regenerated, no files to check against") # Compare json results new_json = _load_json(new_json_file)