allow extensions to add text to the module file

Introduce `make_extension_module_extra` which gets called during module
file creation for every extension similar to `make_module_extra`.
This ensures it will also be called for parallel extension or --module-only builds.

Fixes easybuilders#4647

Author: Flamefire

easybuild/framework/easyblock.py

@@ -1546,21 +1546,30 @@ def make_module_extra_extensions(self):
         Sets optional variables for extensions.
         """
         # add stuff specific to individual extensions
-        lines = [self.module_extra_extensions]
+        txt = self.module_extra_extensions
+
+        if not self.ext_instances:
+            self.prepare_for_extensions()
+            self.init_ext_instances()
+
+        for ext in self.ext_instances:
+            ext_txt = ext.make_extension_module_extra()
+            if ext_txt:
+                txt += ext_txt
 
         # set environment variable that specifies list of extensions
         # We need only name and version, so don't resolve templates
         exts_list = self.make_extension_string(ext_sep=',', sort=False)
         env_var_name = 'EBEXTSLIST' + convert_name(self.name, upper=True)
-        lines.append(self.module_generator.set_environment(env_var_name, exts_list))
+        txt += self.module_generator.set_environment(env_var_name, exts_list)
 
-        return ''.join(lines)
+        return txt
 
     def make_module_footer(self):
         """
         Insert a footer section in the module file, primarily meant for contextual information
         """
-        footer = [self.module_generator.comment("Built with EasyBuild version %s" % VERBOSE_VERSION).rstrip('\n')]
+        footer = []
 
         # add extra stuff for extensions (if any)
         if self.cfg.get_ref('exts_list'):
@@ -1587,6 +1596,7 @@ def make_module_footer(self):
                 self.log.warning("Not including footer in Lua syntax in non-Lua module file: %s",
                                  self.cfg['modluafooter'])
 
+        footer.append(self.module_generator.comment("Built with EasyBuild version %s" % VERBOSE_VERSION))
         return '\n'.join(footer)
 
     def make_module_extend_modpath(self):

easybuild/framework/extension.py

@@ -287,6 +287,10 @@ def toolchain(self):
         """
         return self.master.toolchain
 
+    def make_extension_module_extra(self):
+        """Similar to make_module_extra but only called for extensions"""
+        return ''
+
     def sanity_check_step(self):
         """
         Sanity check to run after installing extension

easybuild/framework/extensioneasyblock.py

@@ -226,6 +226,6 @@ def make_module_extra(self, *args, **kwargs):
             self.log.deprecated("Passing the parameter 'extra' to make_module_extra should be "
                                 "replaced by concatenating the result", '6.0')
         txt = super().make_module_extra(*args, **kwargs)
-        if extra is not None:
+        if extra:
             txt += extra
         return txt

test/framework/easyconfig.py

@@ -482,6 +482,7 @@ def test_exts_list(self):
             homepage = "http://example.com"
             description = "test easyconfig"
             toolchain = SYSTEM
+            exts_defaultclass = "DummyExtension"
             exts_default_options = {
                 "source_tmpl": "gzip-1.4.eb", # dummy source template to avoid download fail
                 "source_urls": ["http://example.com/%(name)s/%(version)s"]

test/framework/sandbox/easybuild/easyblocks/generic/toy_extension.py

@@ -103,6 +103,15 @@ def post_install_extension(self):
 
         EB_toy.install_step(self.master, name=self.name)
 
+    def make_extension_module_extra(self):
+        """Extra stuff for toy extensions"""
+        txt = super(Toy_Extension, self).make_extension_module_extra()
+        value = self.name
+        if self.version:
+            value += '-' + self.version
+        txt += self.module_generator.set_environment('TOY_EXT_VAR', value)
+        return txt
+
     def sanity_check_step(self, *args, **kwargs):
         """Custom sanity check for toy extensions."""
         self.log.info("Loaded modules: %s", self.modules_tool.list())

test/framework/toy_build.py

@@ -1216,6 +1216,8 @@ def test_toy_advanced(self):
             # set by ToyExtension easyblock used to install extensions
             '^setenv.*TOY_EXT_BAR.*bar',
             '^setenv.*TOY_EXT_BARBAR.*barbar',
+            '^setenv.*TOY_EXT_VAR.*bar',
+            '^setenv.*TOY_EXT_VAR.*barbar',
         ]
         for pattern in patterns:
             self.assertTrue(re.search(pattern, toy_mod_txt, re.M), "Pattern '%s' found in: %s" % (pattern, toy_mod_txt))
@@ -1663,8 +1665,8 @@ def test_toy_module_fulltxt(self):
             ] + modloadmsg_lua + [
                 r'end',
                 r'setenv\("TOY", "toy-0.0"\)',
-                r'-- Built with EasyBuild version .*',
                 r'io.stderr:write\("oh hai\!"\)',
+                r'-- Built with EasyBuild version .*',
             ])
         elif get_module_syntax() == 'Tcl':
             mod_txt_regex_pattern = '\n'.join([
@@ -1705,8 +1707,8 @@ def test_toy_module_fulltxt(self):
             ] + modloadmsg_tcl + [
                 r'}',
                 r'setenv	TOY		"toy-0.0"',
-                r'# Built with EasyBuild version .*',
                 r'puts stderr "oh hai\!"',
+                r'# Built with EasyBuild version .*',
             ])
         else:
             self.fail("Unknown module syntax: %s" % get_module_syntax())
@@ -1951,6 +1953,8 @@ def test_module_only_extensions(self):
         with self.mocked_stdout_stderr():
             self.eb_main([test_ec, '--module-only'], do_build=True, raise_error=True)
         self.assertExists(toy_mod)
+        # Extra stuff from extension(s) is included
+        self.assertRegex(read_file(toy_mod), 'TOY_EXT_VAR.*barbar-0.0')
         remove_file(toy_mod)
 
         # rename file required for barbar extension, so we can check whether sanity check catches it
@@ -2008,6 +2012,13 @@ def test_toy_exts_parallel(self):
         stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args, raise_error=True)
         self.assertEqual(stderr, '')
 
+        # Extra stuff from extension(s) should be included
+        ext_var_patterns = [
+            'TOY_EXT_VAR.*ls',
+            'TOY_EXT_VAR.*bar-0.0',
+            'TOY_EXT_VAR.*barbar-0.0',
+        ]
+
         # take into account that each of these lines may appear multiple times,
         # in case no progress was made between checks
         patterns = [
@@ -2022,6 +2033,10 @@ def test_toy_exts_parallel(self):
             error_msg = "Expected pattern '%s' should be found in %s'" % (regex.pattern, stdout)
             self.assertRegex(stdout, regex)
 
+        module_contents = read_file(toy_mod)
+        for pattern in ext_var_patterns:
+            self.assertRegex(module_contents, pattern)
+
         # also test skipping of extensions in parallel
         args.append('--skip')
         stdout, stderr = self.run_test_toy_build_with_output(ec_file=test_ec, extra_args=args, raise_error=True)
@@ -2040,6 +2055,10 @@ def test_toy_exts_parallel(self):
             error_msg = "Expected pattern '%s' should be found in %s'" % (regex.pattern, stdout)
             self.assertTrue(regex.search(stdout), error_msg)
 
+        module_contents = read_file(toy_mod)
+        for pattern in ext_var_patterns:
+            self.assertRegex(module_contents, pattern)
+
         # check behaviour when using Toy_Extension easyblock that doesn't implement required_deps method;
         # framework should fall back to installing extensions sequentially
         toy_ext_eb = os.path.join(topdir, 'sandbox', 'easybuild', 'easyblocks', 'generic', 'toy_extension.py')
@@ -2067,6 +2086,10 @@ def test_toy_exts_parallel(self):
             error_msg = "Expected pattern '%s' should be found in %s'" % (regex.pattern, stdout)
             self.assertTrue(regex.search(stdout), error_msg)
 
+        module_contents = read_file(toy_mod)
+        for pattern in ext_var_patterns:
+            self.assertRegex(module_contents, pattern)
+
     def test_backup_modules(self):
         """Test use of backing up of modules with --module-only."""