diff --git a/easybuild/framework/easyconfig/tweak.py b/easybuild/framework/easyconfig/tweak.py index c3fdfa0904..0229b4e61e 100644 --- a/easybuild/framework/easyconfig/tweak.py +++ b/easybuild/framework/easyconfig/tweak.py @@ -252,7 +252,6 @@ def tweak_one(orig_ec, tweaked_ec, tweaks, targetdir=None): :param targetdir: target directory for tweaked easyconfig file, defaults to temporary directory (only used if tweaked_ec is None) """ - # read easyconfig file ectxt = read_file(orig_ec) @@ -299,7 +298,41 @@ def __repr__(self): for key in list(tweaks): val = tweaks[key] - if isinstance(val, list): + if key in ['dependency', 'builddependency']: + import ast + new_or_updated_deps = [] + for dep in tweaks[key]: + new_or_updated_deps += [tuple(ast.literal_eval(dep))] + + # use non-greedy matching for list value using '*?' to avoid including other parameters in match, + # and a lookahead assertion (?=...) so next line is either another parameter definition or a blank line + param = key.replace('y', 'ies') + regexp = re.compile(r"^(?P\s*%s)\s*=\s*(?P\[(.|\n)*?\])\s*$(?=(\n^\w+\s*=.*|\s*)$)" % param, + re.M) + res = regexp.search(ectxt) + if res: + current_deps = ast.literal_eval(str(res.group('val'))) + + # loop through new or updated deps, if match is found, replace it, else append new + newval = current_deps + for new_dep in new_or_updated_deps: + if new_dep in newval: + continue + + newval = [new_dep if new_dep[0] == curr_dep[0] else curr_dep for curr_dep in newval] + if new_dep not in newval: + _log.debug("Adding dependency %s to %s" % (str(new_dep), param)) + newval += [new_dep] + else: + _log.debug("Updated %s dependency in %s to %s" % (new_dep[0], param, str(new_dep))) + + ectxt = regexp.sub("%s = %s" % (res.group('param'), str(newval)), ectxt) + _log.info("Tweaked %s list to '%s'" % (param, str(newval))) + else: + ectxt += "%s = %s" % (param, str(new_or_updated_deps)) + _log.info("Tweaked %s list to '%s'" % (param, str(new_or_updated_deps))) + + elif isinstance(val, list): # use non-greedy matching for list value using '*?' to avoid including other parameters in match, # and a lookahead assertion (?=...) so next line is either another parameter definition or a blank line regexp = re.compile(r"^(?P\s*%s)\s*=\s*(?P\[(.|\n)*?\])\s*$(?=(\n^\w+\s*=.*|\s*)$)" % key, re.M) @@ -329,6 +362,12 @@ def __repr__(self): tweaks.pop(key) + # these are already handled + if 'dependency' in list(tweaks): + tweaks.pop('dependency') + if 'builddependency' in list(tweaks): + tweaks.pop('builddependency') + # add parameters or replace existing ones special_values = { # if the value is True/False/None then take that diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 6128c3e650..6b120dcd9d 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -300,6 +300,10 @@ def software_options(self): 'amend': (("Specify additional search and build parameters (can be used multiple times); " "for example: versionprefix=foo or patches=one.patch,two.patch)"), None, 'append', None, {'metavar': 'VAR=VALUE[,VALUE]'}), + 'builddependency': ("Specify builddependency to replace or add.", + None, 'append', None, {'metavar': 'VAR=VALUE[,VALUE]'}), + 'dependency': ("Specify dependency to replace or add.", + None, 'append', None, {'metavar': 'VAR=VALUE[,VALUE]'}), 'software': ("Search and build software with given name and version", 'strlist', 'extend', None, {'metavar': 'NAME,VERSION'}), 'software-name': ("Search and build software with given name", @@ -1661,6 +1665,8 @@ def process_software_build_specs(options): 'toolchain_version': options.try_toolchain_version, 'update_deps': options.try_update_deps, 'ignore_versionsuffixes': options.try_ignore_versionsuffixes, + 'dependency': options.try_dependency, + 'builddependency': options.try_builddependency, } # process easy options diff --git a/test/framework/options.py b/test/framework/options.py index 126c52931b..db60b309a8 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1636,6 +1636,91 @@ def test_try_update_deps(self): regex = re.compile(pattern, re.M) self.assertTrue(regex.search(outtxt), "Pattern '%s' should be found in: %s" % (regex.pattern, outtxt)) + def test_try_dependency(self): + """Test for --try-dependency and --try-builddependency.""" + + # first, construct a toy easyconfig that is well suited for testing (multiple deps) + test_ectxt = '\n'.join([ + "easyblock = 'ConfigureMake'", + '', + "name = 'test'", + "version = '1.2.3'", + '' + "homepage = 'https://test.org'", + "description = 'this is just a test'", + '', + "toolchain = {'name': 'GCC', 'version': '4.9.3-2.26'}", + '', + "dependencies = [('hwloc', '1.6.2')]", + ]) + test_ec = os.path.join(self.test_prefix, 'test.eb') + write_file(test_ec, test_ectxt) + + args = [ + test_ec, + '--try-toolchain-version=6.4.0-2.28', + "--try-dependency=('hwloc', '1.11.8')", + "--try-dependency=('FFTW', '3.3.7', '-serial')", + "--disable-map-toolchains", + '-D', + ] + + outtxt = self.eb_main(args, raise_error=True, do_build=True) + patterns = [ + # toolchain got updated + r"^ \* \[x\] .*/test_ecs/g/GCC/GCC-6.4.0-2.28.eb \(module: GCC/6.4.0-2.28\)$", + # hwloc was updated to 1.11.8, thanks to available easyconfig + r"^ \* \[x\] .*/test_ecs/h/hwloc/hwloc-1.11.8-GCC-6.4.0-2.28.eb \(module: hwloc/1.11.8-GCC-6.4.0-2.28\)$", + # FFTW was added, thanks to available easyconfig + r"^ \* \[ \] .*/test_ecs/f/FFTW/FFTW-3.3.7-GCC-6.4.0-2.28-serial.eb " + + r"\(module: FFTW/3.3.7-GCC-6.4.0-2.28-serial\)$", + # also generated easyconfig for test/1.2.3 with expected toolchain + r"^ \* \[ \] .*/tweaked_easyconfigs/test-1.2.3-GCC-6.4.0-2.28.eb \(module: test/1.2.3-GCC-6.4.0-2.28\)$", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(outtxt), "Pattern '%s' should be found in: %s" % (regex.pattern, outtxt)) + + # construct another toy easyconfig that is well suited for testing builddependency + test_ectxt = '\n'.join([ + "easyblock = 'ConfigureMake'", + '', + "name = 'test'", + "version = '1.2.3'", + '' + "homepage = 'https://test.org'", + "description = 'this is just a test'", + '', + "toolchain = {'name': 'GCC', 'version': '4.9.3-2.26'}", + '', + "builddependencies = [('hwloc', '1.6.2')]", + ]) + write_file(test_ec, test_ectxt) + args = [ + test_ec, + '--try-toolchain-version=6.4.0-2.28', + "--try-builddependency=('hwloc', '1.11.8')", + "--try-builddependency=('FFTW', '3.3.7', '-serial')", + "--disable-map-toolchains", + '-D', + ] + outtxt = self.eb_main(args, raise_error=True, do_build=True) + + patterns = [ + # toolchain got updated + r"^ \* \[x\] .*/test_ecs/g/GCC/GCC-6.4.0-2.28.eb \(module: GCC/6.4.0-2.28\)$", + # hwloc was updated to 1.11.8, thanks to available easyconfig + r"^ \* \[x\] .*/test_ecs/h/hwloc/hwloc-1.11.8-GCC-6.4.0-2.28.eb \(module: hwloc/1.11.8-GCC-6.4.0-2.28\)$", + # FFTW was added, thanks to available easyconfig + r"^ \* \[ \] .*/test_ecs/f/FFTW/FFTW-3.3.7-GCC-6.4.0-2.28-serial.eb " + + r"\(module: FFTW/3.3.7-GCC-6.4.0-2.28-serial\)$", + # also generated easyconfig for test/1.2.3 with expected toolchain + r"^ \* \[ \] .*/tweaked_easyconfigs/test-1.2.3-GCC-6.4.0-2.28.eb \(module: test/1.2.3-GCC-6.4.0-2.28\)$", + ] + for pattern in patterns: + regex = re.compile(pattern, re.M) + self.assertTrue(regex.search(outtxt), "Pattern '%s' should be found in: %s" % (regex.pattern, outtxt)) + def test_dry_run_hierarchical(self): """Test dry run using a hierarchical module naming scheme.""" fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')