[WIP]: Python commondata parser

.ciscripts/build-deploy-linux.sh

@@ -6,6 +6,16 @@ set -o pipefail
 set -u
 set -v
 
+# Set the version for both vp and n3fit
+gitroot=$(git rev-parse --show-toplevel)
+gitversion=$(git describe --long --tags)
+tag=$(git describe --abbrev=0 --tags)
+gitversion=${gitversion/${tag}-/${tag}.}
+githash=$(git rev-parse --short HEAD)
+gitversion=${gitversion/-g${githash}/+g${githash}}
+echo "build_version=\"${gitversion}\"" > ${gitroot}/n3fit/src/n3fit/version.py
+echo "build_version=\"${gitversion}\"" > ${gitroot}/validphys2/src/validphys/version.py
+
 #Set up netrc file for uploading/downloading
 echo "$NETRC_FILE" | base64 --decode > ~/.netrc
 
@@ -41,4 +51,4 @@ if [ $? == 0 ]; then
 else
 	echo "Conda package upload failed"
 	exit 1
-fi
+fi

.ciscripts/build-deploy-osx.sh

@@ -5,6 +5,16 @@ set -v
 #Set up netrc file for uploading/downloading
 echo "$NETRC_FILE" | base64 --decode > ~/.netrc
 
+# Set the version for both vp and n3fit
+gitroot=$(git rev-parse --show-toplevel)
+gitversion=$(git describe --long --tags)
+tag=$(git describe --abbrev=0 --tags)
+gitversion=${gitversion/${tag}-/${tag}.}
+githash=$(git rev-parse --short HEAD)
+gitversion=${gitversion/-g${githash}/+g${githash}}
+echo "build_version=\"${gitversion}\"" > ${gitroot}/n3fit/src/n3fit/version.py
+echo "build_version=\"${gitversion}\"" > ${gitroot}/validphys2/src/validphys/version.py
+
 conda build -q conda-recipe
 if [ $? != 0 ]; then
 	echo failed to build

CMakeLists.txt

@@ -42,6 +42,7 @@ option(ENABLE_ASAN "Enable ASAN" OFF)
 option(ENABLE_DEAD_STRIP "Enable use of flag `-dead_strip-dylibs`" OFF)
 option(VP_DEV "validphys in developer mode" ON)
 option(N3_DEV "n3fit in developer mode" ON)
+option(BURN_TAG "burn down the git tag in vp and n3fit (only local non-dev installation)" ON)
 set(PROFILE_PREFIX "" CACHE STRING "Where you store the 'data' folder. Default empty uses CMAKE_INSTALL_PREFIX/share/NNPDF.")
 
 if (PROFILE_PREFIX)
@@ -147,16 +148,54 @@ add_subdirectory(nnpdfcpp)
 # evolven3fit
 add_subdirectory(n3fit/evolven3fit)
 
+if((BURN_TAG) AND (NOT (VP_DEV) OR NOT (N3_DEV)))
+    # Find out where is the root of the git repository (if available) and finds the correct tag to burn down
+    # in the package. This just sets the variable, the actual burning is done down below by the validphys
+    # or n3fit installation and is found in the version.cmake file as it needs to be done post-installation
+    set(GIT_DIR "--git-dir=${PROJECT_SOURCE_DIR}/.git")
+    # get the current tag (ex: 3.4) and check at the same time whether git is available and the .git folder found
+    execute_process(COMMAND git ${GIT_DIR} describe --abbrev=0 --tags OUTPUT_VARIABLE GIT_TAG ERROR_VARIABLE GIT_ERROR)
+    if(GIT_ERROR STREQUAL "")
+        string(STRIP ${GIT_TAG} GIT_TAG)
+        # get the number of revisions since tag happened
+        execute_process(COMMAND git ${GIT_DIR} rev-list ${GIT_TAG}..HEAD --count OUTPUT_VARIABLE GIT_REVN)
+        string(STRIP ${GIT_REVN} GIT_REVN)
+        # get the shortname for the hash
+        execute_process(COMMAND git ${GIT_DIR} rev-parse --short HEAD OUTPUT_VARIABLE GIT_HASH)
+        string(STRIP ${GIT_HASH} GIT_HASH)
+        # find out whether the repository is in a dirty state
+        # (this returns ${GIT_TAG}-dev so we have to remove the tag)
+        execute_process(COMMAND git ${GIT_DIR} describe --abbrev=0 --tags --dirty=-dev OUTPUT_VARIABLE GIT_DIRTY)
+        string(STRIP ${GIT_DIRTY} GIT_DIRTY)
+        string(REPLACE "${GIT_TAG}" "" GIT_DIRTY ${GIT_DIRTY})
+        # now concatenate everything
+        # with the format ex: 3.4.1880+g00b1741c-dev
+        set(GIT_VERSION "build_version='${GIT_TAG}.${GIT_REVN}+g${GIT_HASH}${GIT_DIRTY}'")
+        # now set the variable at install time
+        install(CODE "set(GIT_VERSION \"${GIT_VERSION}\")")
+    else(GIT_ERROR STREQUAL "")
+        set(BURN_TAG OFF)
+    endif(GIT_ERROR STREQUAL "")
+endif()
+
 # install validphys2
 if(VP_DEV)
-  install(CODE "execute_process(COMMAND ${PYTHON_EXECUTABLE} -m pip install -e ${PROJECT_SOURCE_DIR}/validphys2)")
+    install(CODE "execute_process(COMMAND ${PYTHON_EXECUTABLE} -m pip install -e ${PROJECT_SOURCE_DIR}/validphys2)")
 else(VP_DEV)
-  install(CODE "execute_process(COMMAND ${PYTHON_EXECUTABLE} -m pip install --no-deps --ignore-installed ${PROJECT_SOURCE_DIR}/validphys2)")
+    install(CODE "execute_process(COMMAND ${PYTHON_EXECUTABLE} -m pip install --no-deps --ignore-installed ${PROJECT_SOURCE_DIR}/validphys2)")
+    if(BURN_TAG)
+        install(CODE "set(LIBRARY \"validphys\")")
+        install(SCRIPT ${PROJECT_SOURCE_DIR}/version.cmake)
+    endif(BURN_TAG)
 endif(VP_DEV)
 
 # install n3fit
 if(N3_DEV)
     install(CODE "execute_process(COMMAND ${PYTHON_EXECUTABLE} -m pip install -e ${PROJECT_SOURCE_DIR}/n3fit)")
 else(N3_DEV)
     install(CODE "execute_process(COMMAND ${PYTHON_EXECUTABLE} -m pip install --no-deps --ignore-installed ${PROJECT_SOURCE_DIR}/n3fit)")
+    if(BURN_TAG)
+        install(CODE "set(LIBRARY \"n3fit\")")
+        install(SCRIPT ${PROJECT_SOURCE_DIR}/version.cmake)
+    endif(BURN_TAG)
 endif(N3_DEV)

conda-recipe/build.sh

@@ -2,6 +2,6 @@
 
 mkdir build
 cd build
-cmake .. -DCMAKE_INSTALL_PREFIX=${PREFIX} -DVP_DEV=OFF -DN3_DEV=OFF
+cmake .. -DCMAKE_INSTALL_PREFIX=${PREFIX} -DVP_DEV=OFF -DN3_DEV=OFF -DBURN_TAG=OFF
 make -j${CPU_COUNT}
-make install
+make install

conda-recipe/run_test.sh

@@ -3,18 +3,13 @@ set -u
 set -v
 set -e
 
-#Download some resources.
-#This is here to help isolate the errors and avoid timeouts
-vp-get theoryID 162
-vp-get pdf NNPDF31_nnlo_as_0118
-
 #Python tests for the installed validphys package
 pytest --pyargs --mpl validphys
 pytest --pyargs n3fit
 
 mkdir bldtest
 cd bldtest
-cmake .. -DENABLE_TESTS=ON
+cmake .. -DENABLE_TESTS=ON -DBURN_TAG=OFF
 make catch_test -j
 ./libnnpdf/tests/catch_test
 

doc/sphinx/source/get-started/access.md

@@ -60,6 +60,10 @@ Dedicated NNPDF threads for specific Physics projects, which might include non-N
 * [NNPDF_th] For studies and fits related to the implementation of theory uncertainties
 * [NNPDF_jets] For studies related to choice of jet observables and scales
 
+```eval_rst
+.. _NNPDF-server:
+```
+
 ## NNPDF server
 
 *See [Server Configuration](server)*

doc/sphinx/source/tutorials/buildmaster.md

@@ -1,4 +1,4 @@
-# How to Implement a new experiment in buildmaster
+# How to implement a new experiment in buildmaster
 
 Buildmaster is the project that allows the user to generate the ``DATA`` and
 ``SYSTYPE`` files that contain, respectively, the experimental data and the 

doc/sphinx/source/tutorials/index.rst

@@ -6,6 +6,7 @@ Tutorials
 
    ./runafit.md
    ./compare-fits.md
+   ./list-fits.md
    ./report.md
    ./buildmaster.md
    ./APPLgrids.md

doc/sphinx/source/tutorials/list-fits.md

@@ -0,0 +1,8 @@
+# How to list the available fits
+
+The bulk of the available fits can be found by going to the
+[fits folder](https://data.nnpdf.science/fits/) of the NNPDF data server. Some other fits may be
+found in standalone folders in the [home folder](https://data.nnpdf.science/) of this server, but
+of course finding a specific fit here may require some digging. For help in accessing the server,
+please see [here](NNPDF-server). For information on how to download fits and other resources,
+please see the [Downloading resources](download) section of the vp-guide.

n3fit/src/n3fit/__init__.py

@@ -0,0 +1,2 @@
+from n3fit.version import build_version
+__version__ = build_version

n3fit/src/n3fit/version.py

@@ -0,0 +1,41 @@
+# This file gets overwritten on deployemt, don't modify
+def __give_git():
+    # Get the path to this file
+    from pathlib import Path
+
+    file_dir = Path(__file__).parent
+    from subprocess import run, CalledProcessError
+
+    try:
+        result = run(
+            ["git", "describe", "--tags", "--long", "--dirty=-dev"],
+            capture_output=True,
+            text=True,
+            check=True,
+            cwd=file_dir,
+        ).stdout.strip()
+        tag = run(
+            ["git", "describe", "--abbrev=0", "--tags"],
+            capture_output=True,
+            text=True,
+            check=True,
+            cwd=file_dir,
+        ).stdout.strip()
+        githash = run(
+            ["git", "rev-parse", "--short", "HEAD"],
+            capture_output=True,
+            text=True,
+            check=True,
+            cwd=file_dir,
+        ).stdout.strip()
+        version = result.replace(f"-g{githash}", f"+g{githash}").replace(
+            f"{tag}-", f"{tag}."
+        )
+    except CalledProcessError:
+        # In principle this function should not exist on an installed version
+        # but who knows. Also maybe git doesn't work on the machine or whatever
+        version = "unknown"
+    return version
+
+
+build_version = __give_git()

validphys2/src/validphys/__init__.py

@@ -1 +1,3 @@
-#We don't want to import stuff here that could slow down the import times
+#We don't want to import stuff here that could slow down the import times
+from validphys.version import build_version
+__version__ = build_version

validphys2/src/validphys/plotoptions/kintransforms.py

@@ -102,12 +102,63 @@ def xq2map(self, k1, k2, k3, **extra_labels):
 
 class JETXQ2MapMixin:
     def xq2map(self, k1, k2, k3, **extra_labels):
-        """in DY-like experiments k1 is (pseudo)-rapidity and k2 is pT"""
+        """
+        k1 is (pseudo)-rapidity and k2 is pT
+        plotting both x1 and x2
+        """
         ratio = k2/k3
-        x = ratio*(np.exp(k1)+np.exp(-k1))
+        x1 = 2 * ratio * np.exp(k1)
+        x2 = 2 * ratio * np.exp(-k1)
         q2 = k2*k2
-        return np.clip(x,None,1, out=x), q2
+        x = np.concatenate(( x1,x2 ))
+        return np.clip(x,a_min=None,a_max=1, out=x), np.concatenate(( q2,q2 ))
+
+class DIJETXQ2MapMixin:
+    def xq2map(self, k1, k2, k3, **extra_labels):
+        """
+        k1 is max(|y1|,|y2|) and k2 is m12
+        plotting both x1 and x2
+        """
+        ratio = k2/k3
+        x1 = ratio * np.exp(k1)
+        x2 = ratio * np.exp(-k1)
+        q2 = k2*k2
+        x = np.concatenate(( x1,x2 ))
+        return np.clip(x,a_min=None,a_max=1, out=x), np.concatenate(( q2,q2 ))
+
+class DIJETATLASXQ2MapMixin:
+    def xq2map(self, k1, k2, k3, **extra_labels):
+        """
+        k1 is rapidity difference and k2 is m12
+        plotting both x1 and x2
+        """
+        ratio = k2/k3
+        x1 = ratio
+        x2 = np.full_like(x1, 1.0)
+        q2 = k2*k2
+        x = np.concatenate(( x1,x2 ))
+        return np.clip(x,a_min=None,a_max=1, out=x), np.concatenate(( q2,q2 ))
+
+class DIJET3DXQ2MapMixin:
+    def xq2map(self, k1, k2, k3, **extra_labels):
+        """
+        k1 is the rapidity difference, k2 is pTavg, k3 is boost rapidity
+        TODO: NB!! HARDCODING sqrt(s) = 8 TeV, the c.m. energy of 1705.02628.pdf
+        plotting both x1 and x2
+        """
+        sqrts = 8000
+        ratio = k2/sqrts
+        prefactor = ratio * (np.exp(k1) + np.exp(-k1))
+        x1 = prefactor * np.exp(k3)
+        x2 = prefactor * np.exp(-k3)
+        q2 = k2*k2
+        x = np.concatenate(( x1,x2 ))
+        #print(k1[55],k2[55],k3[55],x[55],np.argwhere(x>1))
+        return np.clip(x,a_min=None,a_max=1, out=x), np.concatenate(( q2,q2 ))
+
+
 
+
 class EWPTXQ2MapMixin:
     def xq2map(self, k1, k2, k3, **extra_labels):
         """in ZPt-like Experiments k1 is the pt, k2 is Q"""
@@ -144,18 +195,18 @@ class jet_sqrt_scale(SqrtScaleMixin,JETXQ2MapMixin):
     def new_labels(self, *old_labels):
         return ('$|y|$', '$p_T$ (GeV)', r'$\sqrt{s} (GeV)$')
 
-class dijet_sqrt_scale(SqrtScaleMixin,JETXQ2MapMixin):
+class dijet_sqrt_scale(SqrtScaleMixin,DIJETXQ2MapMixin):
     def new_labels(self, *old_labels):
         return ('$|y|$', '$m_{12}$ (GeV)', r'$\sqrt{s} (GeV)$')
 
-class dijet_sqrt_scale_ATLAS(SqrtScaleMixin,JETXQ2MapMixin):
+class dijet_sqrt_scale_ATLAS(SqrtScaleMixin,DIJETATLASXQ2MapMixin):
     def __call__(self, k1, k2, k3):
         return k1, k2, k3
 
     def new_labels(self, *old_labels):
         return ('$|y^*|$', '$m_{12}$ (TeV)', r'$\sqrt{s} (GeV)$')
 
-class dijet_CMS_3D(SqrtScaleMixin,JETXQ2MapMixin):
+class dijet_CMS_3D(SqrtScaleMixin,DIJET3DXQ2MapMixin):
     def new_labels(self, *old_labels):
         return ('$|y^*|$', '$p_{T,avg}$ (GeV)', r'$|y_b|$')
 

validphys2/src/validphys/version.py

@@ -0,0 +1,41 @@
+# This file gets overwritten on deployemt, don't modify
+def __give_git():
+    # Get the path to this file
+    from pathlib import Path
+
+    file_dir = Path(__file__).parent
+    from subprocess import run, CalledProcessError
+
+    try:
+        result = run(
+            ["git", "describe", "--tags", "--long", "--dirty=-dev"],
+            capture_output=True,
+            text=True,
+            check=True,
+            cwd=file_dir,
+        ).stdout.strip()
+        tag = run(
+            ["git", "describe", "--abbrev=0", "--tags"],
+            capture_output=True,
+            text=True,
+            check=True,
+            cwd=file_dir,
+        ).stdout.strip()
+        githash = run(
+            ["git", "rev-parse", "--short", "HEAD"],
+            capture_output=True,
+            text=True,
+            check=True,
+            cwd=file_dir,
+        ).stdout.strip()
+        version = result.replace(f"-g{githash}", f"+g{githash}").replace(
+            f"{tag}-", f"{tag}."
+        )
+    except CalledProcessError:
+        # In principle this function should not exist on an installed version
+        # but who knows. Also maybe git doesn't work on the machine or whatever
+        version = "unknown"
+    return version
+
+
+build_version = __give_git()

version.cmake

@@ -0,0 +1,3 @@
+execute_process(COMMAND python -c "import ${LIBRARY}.version ; print(${LIBRARY}.version.__file__)" OUTPUT_VARIABLE VERSION_PATH)
+STRING(STRIP ${VERSION_PATH} VERSION_PATH)
+file(WRITE ${VERSION_PATH} ${GIT_VERSION})