diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 38fb3b0e7b..9a6f6265f3 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -177,7 +177,6 @@ def __init__(self, ec, logfile=None): # extensions self.exts = [] - self.exts_all = None self.ext_instances = [] self.skip = None self.module_extra_extensions = '' # extra stuff for module file required by extensions @@ -1546,35 +1545,44 @@ def make_module_extra_extensions(self): Sets optional variables for extensions. """ # add stuff specific to individual extensions - lines = [self.module_extra_extensions] + txt = self.module_extra_extensions + + if not self.ext_instances: + self.prepare_for_extensions() + self.init_ext_instances() + + for ext in self.ext_instances: + ext_txt = ext.make_extension_module_extra() + if ext_txt: + txt += ext_txt # set environment variable that specifies list of extensions # We need only name and version, so don't resolve templates exts_list = self.make_extension_string(ext_sep=',', sort=False) env_var_name = 'EBEXTSLIST' + convert_name(self.name, upper=True) - lines.append(self.module_generator.set_environment(env_var_name, exts_list)) + txt += self.module_generator.set_environment(env_var_name, exts_list) - return ''.join(lines) + return txt def make_module_footer(self): """ Insert a footer section in the module file, primarily meant for contextual information """ - footer = [self.module_generator.comment("Built with EasyBuild version %s" % VERBOSE_VERSION)] + footer = [] # add extra stuff for extensions (if any) if self.cfg.get_ref('exts_list'): footer.append(self.make_module_extra_extensions()) # include modules footer if one is specified - if self.modules_footer is not None: + if self.modules_footer: self.log.debug("Including specified footer into module: '%s'" % self.modules_footer) footer.append(self.modules_footer) if self.cfg['modtclfooter']: if isinstance(self.module_generator, ModuleGeneratorTcl): self.log.debug("Including Tcl footer in module: %s", self.cfg['modtclfooter']) - footer.extend([self.cfg['modtclfooter'], '\n']) + footer.append(self.cfg['modtclfooter']) else: self.log.warning("Not including footer in Tcl syntax in non-Tcl module file: %s", self.cfg['modtclfooter']) @@ -1582,12 +1590,13 @@ def make_module_footer(self): if self.cfg['modluafooter']: if isinstance(self.module_generator, ModuleGeneratorLua): self.log.debug("Including Lua footer in module: %s", self.cfg['modluafooter']) - footer.extend([self.cfg['modluafooter'], '\n']) + footer.append(self.cfg['modluafooter']) else: self.log.warning("Not including footer in Lua syntax in non-Lua module file: %s", self.cfg['modluafooter']) - return ''.join(footer) + footer.append(self.module_generator.comment("Built with EasyBuild version %s" % VERBOSE_VERSION)) + return '\n'.join(footer) def make_module_extend_modpath(self): """ @@ -2139,7 +2148,7 @@ def install_extensions_parallel(self, install=True): running_exts = [] installed_ext_names = [] - all_ext_names = [x['name'] for x in self.exts_all] + all_ext_names = [x['name'] for x in self.exts] self.log.debug("List of names of all extensions: %s", all_ext_names) # take into account that some extensions may be installed already @@ -3090,6 +3099,17 @@ def extensions_step(self, fetch=False, install=True): self.log.debug("No extensions in exts_list") return + self.prepare_for_extensions() + + if fetch: + self.update_exts_progress_bar("fetching extension sources/patches") + self.exts = self.collect_exts_file_info(fetch_files=True) + + # we really need a default class + if not self.cfg['exts_defaultclass']: + raise EasyBuildError("ERROR: No default extension class set for %s", self.name) + self.init_ext_instances() + # load fake module fake_mod_data = None if install and not self.dry_run: @@ -3101,25 +3121,10 @@ def extensions_step(self, fetch=False, install=True): start_progress_bar(PROGRESS_BAR_EXTENSIONS, len(self.cfg.get_ref('exts_list'))) - self.prepare_for_extensions() - - if fetch: - self.update_exts_progress_bar("fetching extension sources/patches") - self.exts = self.collect_exts_file_info(fetch_files=True) - - self.exts_all = self.exts[:] # retain a copy of all extensions, regardless of filtering/skipping - # actually install extensions if install: self.log.info("Installing extensions") - # we really need a default class - if not self.cfg['exts_defaultclass'] and fake_mod_data: - self.clean_up_fake_module(fake_mod_data) - raise EasyBuildError("ERROR: No default extension class set for %s", self.name) - - self.init_ext_instances() - if self.skip: self.skip_extensions() diff --git a/easybuild/framework/extension.py b/easybuild/framework/extension.py index 7471670a90..2f7a6163fc 100644 --- a/easybuild/framework/extension.py +++ b/easybuild/framework/extension.py @@ -287,6 +287,10 @@ def toolchain(self): """ return self.master.toolchain + def make_extension_module_extra(self): + """Similar to make_module_extra but only called for extensions""" + return '' + def sanity_check_step(self): """ Sanity check to run after installing extension diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index b85b97265a..5a1cc22ba3 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -226,6 +226,6 @@ def make_module_extra(self, *args, **kwargs): self.log.deprecated("Passing the parameter 'extra' to make_module_extra should be " "replaced by concatenating the result", '6.0') txt = super().make_module_extra(*args, **kwargs) - if extra is not None: + if extra: txt += extra return txt diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index c4f4ce4439..9f22a70db2 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -482,6 +482,7 @@ def test_exts_list(self): homepage = "http://example.com" description = "test easyconfig" toolchain = SYSTEM + exts_defaultclass = "DummyExtension" exts_default_options = { "source_tmpl": "gzip-1.4.eb", # dummy source template to avoid download fail "source_urls": ["http://example.com/%(name)s/%(version)s"] diff --git a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py index c347e05d40..8780d55b5f 100644 --- a/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py +++ b/test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py @@ -103,6 +103,15 @@ def post_install_extension(self): EB_toy.install_step(self.master, name=self.name) + def make_extension_module_extra(self): + """Extra stuff for toy extensions""" + txt = super(Toy_Extension, self).make_extension_module_extra() + value = self.name + if self.version: + value += '-' + self.version + txt += self.module_generator.set_environment('TOY_EXT_VAR', value) + return txt + def sanity_check_step(self, *args, **kwargs): """Custom sanity check for toy extensions.""" self.log.info("Loaded modules: %s", self.modules_tool.list()) diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 5f02ea2f1e..9d873dea3f 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1216,6 +1216,8 @@ def test_toy_advanced(self): # set by ToyExtension easyblock used to install extensions '^setenv.*TOY_EXT_BAR.*bar', '^setenv.*TOY_EXT_BARBAR.*barbar', + '^setenv.*TOY_EXT_VAR.*bar', + '^setenv.*TOY_EXT_VAR.*barbar', ] for pattern in patterns: self.assertTrue(re.search(pattern, toy_mod_txt, re.M), "Pattern '%s' found in: %s" % (pattern, toy_mod_txt)) @@ -1663,8 +1665,8 @@ def test_toy_module_fulltxt(self): ] + modloadmsg_lua + [ r'end', r'setenv\("TOY", "toy-0.0"\)', + r'io.stderr:write\("oh hai\!"\)', r'-- Built with EasyBuild version .*', - r'io.stderr:write\("oh hai\!"\)$', ]) elif get_module_syntax() == 'Tcl': mod_txt_regex_pattern = '\n'.join([ @@ -1705,15 +1707,13 @@ def test_toy_module_fulltxt(self): ] + modloadmsg_tcl + [ r'}', r'setenv TOY "toy-0.0"', + r'puts stderr "oh hai\!"', r'# Built with EasyBuild version .*', - r'puts stderr "oh hai\!"$', ]) else: self.fail("Unknown module syntax: %s" % get_module_syntax()) - mod_txt_regex = re.compile(mod_txt_regex_pattern) - msg = "Pattern '%s' matches with: %s" % (mod_txt_regex.pattern, toy_mod_txt) - self.assertTrue(mod_txt_regex.match(toy_mod_txt), msg) + self.assertRegex(toy_mod_txt, mod_txt_regex_pattern) def test_external_dependencies(self): """Test specifying external (build) dependencies.""" @@ -1953,6 +1953,8 @@ def test_module_only_extensions(self): with self.mocked_stdout_stderr(): self.eb_main([test_ec, '--module-only'], do_build=True, raise_error=True) self.assertExists(toy_mod) + # Extra stuff from extension(s) is included + self.assertRegex(read_file(toy_mod), 'TOY_EXT_VAR.*barbar-0.0') remove_file(toy_mod) # rename file required for barbar extension, so we can check whether sanity check catches it @@ -2010,6 +2012,13 @@ def test_toy_exts_parallel(self): stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args, raise_error=True) self.assertEqual(stderr, '') + # Extra stuff from extension(s) should be included + ext_var_patterns = [ + 'TOY_EXT_VAR.*ls', + 'TOY_EXT_VAR.*bar-0.0', + 'TOY_EXT_VAR.*barbar-0.0', + ] + # take into account that each of these lines may appear multiple times, # in case no progress was made between checks patterns = [ @@ -2022,7 +2031,11 @@ def test_toy_exts_parallel(self): for pattern in patterns: regex = re.compile(pattern, re.M) error_msg = "Expected pattern '%s' should be found in %s'" % (regex.pattern, stdout) - self.assertTrue(regex.search(stdout), error_msg) + self.assertRegex(stdout, regex) + + module_contents = read_file(toy_mod) + for pattern in ext_var_patterns: + self.assertRegex(module_contents, pattern) # also test skipping of extensions in parallel args.append('--skip') @@ -2042,6 +2055,10 @@ def test_toy_exts_parallel(self): error_msg = "Expected pattern '%s' should be found in %s'" % (regex.pattern, stdout) self.assertTrue(regex.search(stdout), error_msg) + module_contents = read_file(toy_mod) + for pattern in ext_var_patterns: + self.assertRegex(module_contents, pattern) + # check behaviour when using Toy_Extension easyblock that doesn't implement required_deps method; # framework should fall back to installing extensions sequentially toy_ext_eb = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks', 'generic', 'toy_extension.py') @@ -2069,6 +2086,10 @@ def test_toy_exts_parallel(self): error_msg = "Expected pattern '%s' should be found in %s'" % (regex.pattern, stdout) self.assertTrue(regex.search(stdout), error_msg) + module_contents = read_file(toy_mod) + for pattern in ext_var_patterns: + self.assertRegex(module_contents, pattern) + def test_backup_modules(self): """Test use of backing up of modules with --module-only.""" @@ -3328,13 +3349,13 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): ' gcc toy.c -o toy && copy_toy_file toy copy_of_toy' command failed (exit code 127), but I fixed it! in post-install hook for toy v0.0 bin, lib - in module-write hook hook for {mod_name} toy 0.0 ['%(name)s-%(version)s.tar.gz'] echo toy toy 0.0 ['%(name)s-%(version)s.tar.gz'] echo toy + in module-write hook hook for {mod_name} installing of extension bar is done! pre_run_shell_cmd_hook triggered for ' gcc toy.c -o toy ' ' gcc toy.c -o toy && copy_toy_file toy copy_of_toy' command failed (exit code 127), but I fixed it!