diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 09c35fbf4d..b583b7ae8a 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -60,6 +60,7 @@ else fi TMPDIR=$(mktemp -d) +echo "TMPDIR set to '${TMPDIR}'" echo ">> Setting up environment..." @@ -80,6 +81,9 @@ fi # avoid that pyc files for EasyBuild are stored in EasyBuild installation directory export PYTHONPYCACHEPREFIX=$TMPDIR/pycache +echo "check PYTHONPYCACHEPREFIX (${PYTHONPYCACHEPREFIX})" +ls -l $TMPDIR +# mkdir -p ${PYTHONPYCACHEPREFIX} DETECTION_PARAMETERS='' GENERIC=0 @@ -177,10 +181,6 @@ else eb --install-latest-eb-release &> ${eb_install_out} check_exit_code $? "${ok_msg}" "${fail_msg}" - # restore origin $PATH and $PYTHONPATH values - export PATH=${ORIG_PATH} - export PYTHONPATH=${ORIG_PYTHONPATH} - eb --search EasyBuild-${REQ_EB_VERSION}.eb | grep EasyBuild-${REQ_EB_VERSION}.eb > /dev/null if [[ $? -eq 0 ]]; then ok_msg="EasyBuild v${REQ_EB_VERSION} installed, alright!" @@ -189,6 +189,10 @@ else check_exit_code $? "${ok_msg}" "${fail_msg}" fi + # restore origin $PATH and $PYTHONPATH values + export PATH=${ORIG_PATH} + export PYTHONPATH=${ORIG_PYTHONPATH} + module avail easybuild/${REQ_EB_VERSION} &> ${ml_av_easybuild_out} if [[ $? -eq 0 ]]; then echo_green ">> EasyBuild module installed!" @@ -281,11 +285,11 @@ echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALL #$EB Perl-5.30.2-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2640 #check_exit_code $? "${ok_msg}" "${fail_msg}" -echo ">> Installing Qt5..." -ok_msg="Qt5 installed, phieuw, that was a big one!" -fail_msg="Installation of Qt5 failed, that's frustrating..." -$EB Qt5-5.14.1-GCCcore-9.3.0.eb --robot --disable-cleanup-tmpdir -check_exit_code $? "${ok_msg}" "${fail_msg}" +#echo ">> Installing Qt5..." +#ok_msg="Qt5 installed, phieuw, that was a big one!" +#fail_msg="Installation of Qt5 failed, that's frustrating..." +#$EB Qt5-5.14.1-GCCcore-9.3.0.eb --robot --disable-cleanup-tmpdir +#check_exit_code $? "${ok_msg}" "${fail_msg}" # skip test step when installing SciPy-bundle on aarch64, # to dance around problem with broken numpy tests; @@ -368,11 +372,11 @@ check_exit_code $? "${ok_msg}" "${fail_msg}" #$EB OSU-Micro-Benchmarks-5.6.3-gompi-2020a.eb -r #check_exit_code $? "${ok_msg}" "${fail_msg}" -echo ">> Installing Spark 3.1.1..." -ok_msg="Spark installed, set off the fireworks!" -fail_msg="Installation of Spark failed, no fireworks this time..." -$EB Spark-3.1.1-foss-2020a-Python-3.8.2.eb -r -check_exit_code $? "${ok_msg}" "${fail_msg}" +#echo ">> Installing Spark 3.1.1..." +#ok_msg="Spark installed, set off the fireworks!" +#fail_msg="Installation of Spark failed, no fireworks this time..." +#$EB Spark-3.1.1-foss-2020a-Python-3.8.2.eb -r +#check_exit_code $? "${ok_msg}" "${fail_msg}" #echo ">> Installing IPython 7.15.0..." #ok_msg="IPython installed, launch your Jupyter Notebooks!" @@ -392,12 +396,12 @@ check_exit_code $? "${ok_msg}" "${fail_msg}" #$EB --from-pr 14821 X11-20210518-GCCcore-10.3.0.eb -r && $EB --from-pr 16011 R-4.1.0-foss-2021a.eb --robot --parallel-extensions-install --experimental #check_exit_code $? "${ok_msg}" "${fail_msg}" -echo ">> Installing Nextflow 22.10.1..." -ok_msg="Nextflow installed, the work must flow..." -fail_msg="Installation of Nextflow failed, that's unexpected..." -# Comment from Axel: PR 16531 was merged so --from-pr not needed anymore (but was used in this build) -$EB -r --from-pr 16531 Nextflow-22.10.1.eb -check_exit_code $? "${ok_msg}" "${fail_msg}" +#echo ">> Installing Nextflow 22.10.1..." +#ok_msg="Nextflow installed, the work must flow..." +#fail_msg="Installation of Nextflow failed, that's unexpected..." +## Comment from Axel: PR 16531 was merged so --from-pr not needed anymore (but was used in this build) +#$EB -r --from-pr 16531 Nextflow-22.10.1.eb +#check_exit_code $? "${ok_msg}" "${fail_msg}" #echo ">> Installing OSU-Micro-Benchmarks/5.7.1-gompi-2021a..." #ok_msg="OSU-Micro-Benchmarks installed, yihaa!" @@ -425,7 +429,7 @@ check_exit_code $? "${ok_msg}" "${fail_msg}" #$EB Perl-5.32.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2640 #use enhanced CMake easyblock to patch CMake's UnixPaths.cmake script if --sysroot is set #from https://github.com/easybuilders/easybuild-easyblocks/pull/2248 -$EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 +#$EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 # use Rust easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14584 # that includes patch to fix bootstrap problem when using alternate sysroot #$EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot @@ -450,10 +454,10 @@ $EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 ##################### ### add packages here ##################### -$EB Python-3.9.5-GCCcore-10.3.0.eb --robot -$EB OpenMPI-4.1.1-GCC-10.3.0.eb --robot +#$EB Python-3.9.5-GCCcore-10.3.0.eb --robot +#$EB OpenMPI-4.1.1-GCC-10.3.0.eb --robot # this Package has been added to reduce the complexity of building large packages such as R -$EB ImageMagick-7.0.11-14-GCCcore-10.3.0.eb --robot +#$EB ImageMagick-7.0.11-14-GCCcore-10.3.0.eb --robot # example block showing a few debugging means #echo "Installing CaDiCaL/1.3.0 for GCC/9.3.0..." #ok_msg="CaDiCaL installed. Nice!" @@ -465,17 +469,34 @@ $EB ImageMagick-7.0.11-14-GCCcore-10.3.0.eb --robot #check_exit_code $exit_code "${ok_msg}" "${fail_msg}" # add latest EasyBuild to stack -echo ">> Adding latest EasyBuild to stack..." -ok_msg="Latest EasyBuild got installed ... great!" -fail_msg="Installation of latest EasyBuild failed! Disappointed." -if [[ ${EESSI_CVMFS_REPO} == /cvmfs/pilot.eessi-hpc.org ]]; then - $EB --from-pr 14545 --include-easyblocks-from-pr 2805 --robot --install-latest-eb-release -else - $EB --robot --install-latest-eb-release -fi -exit_code=$? -check_exit_code ${exit_code} "${ok_msg}" "${fail_msg}" +#echo ">> Adding latest EasyBuild to stack..." +#ok_msg="Latest EasyBuild got installed ... great!" +#fail_msg="Installation of latest EasyBuild failed! Disappointed." +#if [[ ${EESSI_CVMFS_REPO} == /cvmfs/pilot.eessi-hpc.org ]]; then +# $EB --from-pr 14545 --include-easyblocks-from-pr 2805 --robot --install-latest-eb-release +##else +# $EB --robot --install-latest-eb-release +#fi +#exit_code=$? +#check_exit_code ${exit_code} "${ok_msg}" "${fail_msg}" + +echo "BEFORE EB-installing BLIS" +ls -l ${EESSI_SOFTWARE_PATH}/software/Python/3.9.5-GCCcore-10.3.0/easybuild/python/__pycache__/ + +$EB --robot BLIS-0.8.1-GCC-10.3.0.eb + +echo "AFTER EB-installing BLIS" +ls -l ${EESSI_SOFTWARE_PATH}/software/Python/3.9.5-GCCcore-10.3.0/easybuild/python/__pycache__/ + +$EB --robot OpenBLAS-0.3.15-GCC-10.3.0.eb + +echo "AFTER EB-installing OpenBLAS" +ls -l ${EESSI_SOFTWARE_PATH}/software/Python/3.9.5-GCCcore-10.3.0/easybuild/python/__pycache__/ + +$EB --robot FlexiBLAS-3.0.4-GCC-10.3.0.eb +echo "AFTER EB-installing FlexiBLAS" +ls -l ${EESSI_SOFTWARE_PATH}/software/Python/3.9.5-GCCcore-10.3.0/easybuild/python/__pycache__/ echo ">> Creating/updating Lmod cache..." export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua" diff --git a/create_tarball.sh b/create_tarball.sh index 56ac8ab7ad..08dc4f4d7e 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -38,6 +38,7 @@ cd ${overlay_upper_dir}/versions/ echo ">> Collecting list of files/directories to include in tarball via ${PWD}..." files_list=${tmpdir}/files.list.txt +module_files_list=${tmpdir}/module_files.list.txt if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then # include Lmod cache and configuration file (lmodrc.lua), @@ -49,12 +50,26 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} # module symlinks find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} + # module files and symlinks + find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f -o -type l \ + | grep -v '/\.wh\.' | sed -e 's/.lua$//' | awk -F'/' '{printf "%s/%s\n", $(NF-1), $NF}' | sort | uniq \ + >> ${module_files_list} fi if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software ]; then # installation directories - ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/*/* | grep -v '/\.wh\.' >> ${files_list} + # ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/*/* | grep -v '/\.wh\.' >> ${files_list} + for package_version in $(cat ${module_files_list}); do + echo "handling ${package_version}" + ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/${package_version} \ + | grep -v '/\.wh\.' >> ${files_list} + done fi +echo "wrote file list to ${files_list}" +cat ${files_list} +echo "wrote module file list to ${module_files_list}" +cat ${module_files_list} + topdir=${cvmfs_repo}/versions/ echo ">> Creating tarball ${target_tgz} from ${topdir}..."